removed hardcoded strings and added constants

This commit is contained in:
Viet Huynh
2023-09-18 04:47:26 +07:00
parent 0657dc7e94
commit fa279ee24f
4 changed files with 40 additions and 15 deletions
+7 -6
View File
@@ -10,6 +10,7 @@ import {
FAST_MODE, FAST_MODE,
LIST_ITEM_LIMIT, LIST_ITEM_LIMIT,
LIST_ITEM_SIZE, LIST_ITEM_SIZE,
PROCESSING_FILENAME,
} from "./lib/constants.js"; } from "./lib/constants.js";
import { normalizeDomain } from "./lib/helpers.js"; import { normalizeDomain } from "./lib/helpers.js";
import { import {
@@ -19,12 +20,12 @@ import {
readFile, readFile,
} from "./lib/utils.js"; } from "./lib/utils.js";
const allowlistFilename = existsSync("whitelist.csv") const allowlistFilename = existsSync(PROCESSING_FILENAME.OLD_ALLOWLIST)
? "whitelist.csv" ? PROCESSING_FILENAME.OLD_ALLOWLIST
: "allowlist.txt"; : PROCESSING_FILENAME.ALLOWLIST;
const blocklistFilename = existsSync("input.csv") const blocklistFilename = existsSync(PROCESSING_FILENAME.OLD_BLOCKLIST)
? "input.csv" ? PROCESSING_FILENAME.OLD_BLOCKLIST
: "blocklist.txt"; : PROCESSING_FILENAME.BLOCKLIST;
const allowlist = new Map(); const allowlist = new Map();
const blocklist = new Map(); const blocklist = new Map();
const domains = []; const domains = [];
+13 -6
View File
@@ -1,5 +1,6 @@
import { resolve } from "path"; import { resolve } from "path";
import { LIST_TYPE, PROCESSING_FILENAME } from "./lib/constants.js";
import { downloadFiles } from "./lib/utils.js"; import { downloadFiles } from "./lib/utils.js";
const allowlistUrls = [ const allowlistUrls = [
@@ -40,25 +41,31 @@ const blocklistUrls = [
const listType = process.argv[2]; const listType = process.argv[2];
const downloadAllowlists = async () => { const downloadAllowlists = async () => {
await downloadFiles(resolve("./allowlist.txt"), allowlistUrls); await downloadFiles(
resolve(`./${PROCESSING_FILENAME.ALLOWLIST}`),
allowlistUrls
);
console.log( console.log(
"Done. The allowlist.txt file contains merged data from recommended whitelists." `Done. The ${PROCESSING_FILENAME.ALLOWLIST} file contains merged data from recommended allowlists.`
); );
}; };
const downloadBlocklists = async () => { const downloadBlocklists = async () => {
await downloadFiles(resolve("./blocklist.txt"), blocklistUrls); await downloadFiles(
resolve(`./${PROCESSING_FILENAME.BLOCKLIST}`),
blocklistUrls
);
console.log( console.log(
"Done. The blocklist.txt file contains merged data from recommended filter lists." `Done. The ${PROCESSING_FILENAME.BLOCKLIST} file contains merged data from recommended blocklists.`
); );
}; };
switch (listType) { switch (listType) {
case "allowlist": { case LIST_TYPE.ALLOWLIST: {
await downloadAllowlists(); await downloadAllowlists();
break; break;
} }
case "blocklist": { case LIST_TYPE.BLOCKLIST: {
await downloadBlocklists(); await downloadBlocklists();
break; break;
} }
+12
View File
@@ -19,3 +19,15 @@ export const API_HOST = "https://api.cloudflare.com/client/v4";
export const DRY_RUN = !!parseInt(process.env.DRY_RUN, 10); export const DRY_RUN = !!parseInt(process.env.DRY_RUN, 10);
export const FAST_MODE = !!parseInt(process.env.FAST_MODE, 10); export const FAST_MODE = !!parseInt(process.env.FAST_MODE, 10);
export const PROCESSING_FILENAME = {
ALLOWLIST: "allowlist.txt",
BLOCKLIST: "blocklist.txt",
OLD_ALLOWLIST: "whitelist.csv",
OLD_BLOCKLIST: "input.csv",
};
export const LIST_TYPE = {
ALLOWLIST: "allowlist",
BLOCKLIST: "blocklist",
};
+8 -3
View File
@@ -1,9 +1,14 @@
import { ACCOUNT_EMAIL, ACCOUNT_ID, API_HOST, API_TOKEN } from "./constants.js"; import { ACCOUNT_EMAIL, ACCOUNT_ID, API_HOST, API_TOKEN } from "./constants.js";
if (!globalThis.fetch) { if (!globalThis.fetch) {
console.warn("\nIMPORTANT: Your Node.js version doesn't have native fetch support and may not be supported in the future. Please update to v18 or later.\n") console.warn(
"\nIMPORTANT: Your Node.js version doesn't have native fetch support and may not be supported in the future. Please update to v18 or later.\n"
);
// Advise what to do if running in GitHub Actions // Advise what to do if running in GitHub Actions
if (process.env.GITHUB_WORKSPACE) console.warn("Since you're running in GitHub Actions, you should update your Actions workflow configuration to use Node v18 or higher.") if (process.env.GITHUB_WORKSPACE)
console.warn(
"Since you're running in GitHub Actions, you should update your Actions workflow configuration to use Node v18 or higher."
);
// Import node-fetch since there's no native fetch in this environment // Import node-fetch since there's no native fetch in this environment
globalThis.fetch = (await import("node-fetch")).default; globalThis.fetch = (await import("node-fetch")).default;
} }
@@ -48,7 +53,7 @@ export const requestGateway = (path, options) =>
/** /**
* Normalizes a domain. * Normalizes a domain.
* @param {string} value The value to be normalized. * @param {string} value The value to be normalized.
* @param {boolean} isAllowlisting Whether the value is to be whitelisted. * @param {boolean} isAllowlisting Whether the value is to be allowlisted.
* @returns {string} * @returns {string}
*/ */
export const normalizeDomain = (value, isAllowlisting) => { export const normalizeDomain = (value, isAllowlisting) => {