use native fetch and refactored list creation

.env.example

@@ -1,4 +1,4 @@
-CLOUDFLARE_API_KEY=""
+CLOUDFLARE_API_KEY=
-CLOUDFLARE_ACCOUNT_ID=""
+CLOUDFLARE_ACCOUNT_ID=
-CLOUDFLARE_ACCOUNT_EMAIL=""
+CLOUDFLARE_ACCOUNT_EMAIL=
-CLOUDFLARE_LIST_ITEM_LIMIT="300000"
+CLOUDFLARE_LIST_ITEM_LIMIT=300000

cf_list_create.js

@@ -1,11 +1,6 @@
-import 'dotenv/config';
-import fetch from 'node-fetch';
 import fs from 'fs';
-
+import { DRY_RUN, LIST_ITEM_LIMIT } from './lib/constants.js';
-const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
+import { createZeroTrustLists } from './lib/helpers.js';
-const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
-const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
-const LIST_ITEM_LIMIT = Number.isSafeInteger(Number(process.env.CLOUDFLARE_LIST_ITEM_LIMIT)) ? Number(process.env.CLOUDFLARE_LIST_ITEM_LIMIT) : 300000;
 
 if (!process.env.CI) console.log(`List item limit set to ${LIST_ITEM_LIMIT}`);
 
@@ -113,79 +108,11 @@ fs.readFile('input.csv', 'utf8', async (err, data) => {
 
   // If we are dry-running, stop here because we don't want to create lists
   // TODO: we should probably continue, just without making any real requests to Cloudflare
-  if (process.env.DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.');
+  if (DRY_RUN) return console.log('Dry run complete - no lists were created. If this was not intended, please remove the DRY_RUN environment variable and try again.');
 
-  // Separate domains into chunks of 1000 (Cloudflare list cap)
+  await createZeroTrustLists(domains)
-  const chunks = chunkArray(domains, 1000);
-
-  // Create Cloudflare Zero Trust lists
-  for (const [index, chunk] of chunks.entries()) {
-    const listName = `CGPS List - Chunk ${index}`;
-
-    let properList = [];
-
-    chunk.forEach(domain => {
-        properList.push({ "value": domain })
-    });
-
-    try {
-      await createZeroTrustList(listName, properList, (index+1), listsToCreate);
-      await sleep(350); // Sleep for 350ms between list additions
-    } catch (error) {
-      console.error(`Error creating list `, process.env.CI ? "(redacted on CI)" :  `"${listName}": ${error.response.data}`);
-    }
-  }
 });
 
 function trimArray(arr, size) {
   return arr.slice(0, size);
 }
-
-// Function to check if a domain is valid
-function isValidDomain(domain) {
-    const regex = /^((?!-)[A-Za-z0-9-]{1,63}(?<!-)\.)+[A-Za-z]{2,6}$/;
-    return regex.test(domain);
-}
-
-// Function to split an array into chunks
-function chunkArray(array, chunkSize) {
-  const chunks = [];
-  for (let i = 0; i < array.length; i += chunkSize) {
-    chunks.push(array.slice(i, i + chunkSize));
-  }
-  return chunks;
-}
-
-// Function to create a Cloudflare Zero Trust list
-async function createZeroTrustList(name, items, currentItem, totalItems) {
-  const url = `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/gateway/lists`;
-
-  const response = await fetch(url, {
-    method: 'POST',
-    headers: {
-      'Authorization': `Bearer ${API_TOKEN}`,
-      'Content-Type': 'application/json',
-      'X-Auth-Email': ACCOUNT_EMAIL,
-      'X-Auth-Key': API_TOKEN,
-    },
-    body: JSON.stringify({
-      name,
-      type: 'DOMAIN', // Set list type to DOMAIN
-      items,
-    }),
-  });
-
-  const data = await response.json();
-  const listId = data.result.id;
-
-  console.log(`Created Zero Trust list`, process.env.CI ? "(redacted on CI)" : `"${name}" with ID ${listId} - ${totalItems - currentItem} left`);
-}
-
-function percentage(percent, total) {
-  return Math.round((percent / 100) * total);
-}
-
-// Function to sleep for a specified duration
-function sleep(ms) {
-  return new Promise(resolve => setTimeout(resolve, ms));
-}

lib/constants.js

@@ -0,0 +1,25 @@
+import { isDev } from "./utils.js";
+
+if (isDev()) {
+  const dotenv = await import("dotenv");
+
+  dotenv.config();
+}
+
+export const API_TOKEN = process.env.CLOUDFLARE_API_KEY;
+
+export const ACCOUNT_ID = process.env.CLOUDFLARE_ACCOUNT_ID;
+
+export const ACCOUNT_EMAIL = process.env.CLOUDFLARE_ACCOUNT_EMAIL;
+
+export const LIST_ITEM_LIMIT = isNaN(process.env.CLOUDFLARE_LIST_ITEM_LIMIT)
+  ? 300000
+  : parseInt(process.env.CLOUDFLARE_LIST_ITEM_LIMIT, 10);
+
+export const LIST_ITEM_SIZE = 1000;
+
+export const API_HOST = "https://api.cloudflare.com/client/v4";
+
+export const DRY_RUN = !!parseInt(process.env.DRY_RUN, 10);
+
+export const FAST_MODE = !!parseInt(process.env.FAST_MODE, 10);

lib/helpers.js

@@ -0,0 +1,89 @@
+import {
+  ACCOUNT_EMAIL,
+  ACCOUNT_ID,
+  API_HOST,
+  API_TOKEN,
+  FAST_MODE,
+  LIST_ITEM_SIZE,
+} from "./constants.js";
+import { sleep } from "./utils.js";
+
+const request = async (url, options) => {
+  const response = await fetch(url, {
+    method: "GET",
+    headers: {
+      Authorization: `Bearer ${API_TOKEN}`,
+      "Content-Type": "application/json",
+      "X-Auth-Email": ACCOUNT_EMAIL,
+      "X-Auth-Key": API_TOKEN,
+    },
+    ...options,
+  });
+
+  return response.json();
+};
+
+const createZeroTrustList = (name, items) => {
+  // https://developers.cloudflare.com/api/operations/zero-trust-lists-create-zero-trust-list
+
+  return request(`${API_HOST}/accounts/${ACCOUNT_ID}/gateway/lists`, {
+    method: "POST",
+    body: JSON.stringify({
+      name,
+      type: "DOMAIN",
+      items,
+    }),
+  });
+};
+
+const createZeroTrustListsOneByOne = async (items) => {
+  let totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE);
+
+  for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) {
+    const chunk = items
+      .slice(i, i + LIST_ITEM_SIZE)
+      .map((item) => ({ value: item }));
+    const listName = `CGPS List - Chunk ${listNumber}`;
+
+    try {
+      const {
+        result: { id },
+      } = await createZeroTrustList(listName, chunk);
+
+      totalListNumber--;
+      listNumber++;
+      console.log(
+        `Created ${listName} with ID ${id} - ${totalListNumber} left`
+      );
+      await sleep();
+    } catch (err) {
+      console.error(`Could not create ${listName} - ${err.toString()}`);
+    }
+  }
+};
+
+const createZeroTrustListsAtOnce = async (items) => {
+  let totalListNumber = Math.ceil(items.length / LIST_ITEM_SIZE);
+  const requests = [];
+
+  for (let i = 0, listNumber = 1; i < items.length; i += LIST_ITEM_SIZE) {
+    const chunk = items
+      .slice(i, i + LIST_ITEM_SIZE)
+      .map((item) => ({ value: item }));
+    const listName = `CGPS List - Chunk ${listNumber}`;
+
+    requests.push(createZeroTrustList(listName, chunk));
+    listNumber++;
+  }
+
+  await Promise.all(requests);
+  console.log(`Created ${totalListNumber} lists`);
+};
+
+export const createZeroTrustLists = (items) => {
+  if (FAST_MODE) {
+    return createZeroTrustListsAtOnce(items);
+  }
+
+  return createZeroTrustListsOneByOne(items);
+};

lib/utils.js

@@ -0,0 +1,4 @@
+export const isDev = () => process.env.NODE_ENV !== "production";
+
+export const sleep = (ms = 350) =>
+  new Promise((resolve) => setTimeout(resolve, ms));