From 053a5e86b26bf541cc163a27b94d7ead126c2a53 Mon Sep 17 00:00:00 2001 From: Norbert Date: Fri, 30 Jun 2023 07:36:37 +0200 Subject: [PATCH 1/8] Update whitelist command: Display domain count --- cf_list_create.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cf_list_create.js b/cf_list_create.js index 738af4f..af80e41 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -38,10 +38,10 @@ fs.readFile('whitelist.csv', 'utf8', async (err, data) => { }).filter(domain => { return domainValidationPattern.test(domain); }); + console.log(`Found ${whitelist.length} valid domains in whitelist.`); } }); -console.log(`Found ${whitelist.length} valid domains in whitelist.`); // Read input.csv and parse domains fs.readFile('input.csv', 'utf8', async (err, data) => { From 39dd3a3d0f15d56de19dac9edfb79168b8f7fa35 Mon Sep 17 00:00:00 2001 From: Norbert Date: Fri, 30 Jun 2023 07:53:13 +0200 Subject: [PATCH 2/8] console log whitelist: ${domain} - removing --- cf_list_create.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cf_list_create.js b/cf_list_create.js index af80e41..dd77c19 100644 --- a/cf_list_create.js +++ b/cf_list_create.js @@ -90,7 +90,11 @@ fs.readFile('input.csv', 'utf8', async (err, data) => { // Remove domains from the domains array that are present in the whitelist array domains = domains.filter(domain => { - return !whitelist.includes(domain); + if (whitelist.includes(domain)) { + console.warn(`Domain found in the whitelist: ${domain} - removing`); + return false; + } + return true; }); // Trim array to 300,000 domains if it's longer than that From c7fc250f78aad6f24b0c1d77fef196a93a7de6ff Mon Sep 17 00:00:00 2001 From: Norbert Date: Fri, 30 Jun 2023 09:23:12 +0200 Subject: [PATCH 3/8] whitelist log test --- get_recommended_whitelist.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/get_recommended_whitelist.sh b/get_recommended_whitelist.sh index 74c4c92..4d2ede4 100644 --- a/get_recommended_whitelist.sh +++ b/get_recommended_whitelist.sh @@ -7,6 +7,7 @@ touch whitelist.csv # declare an array of urls urls=( + https://adaway.org/hosts.txt #test https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist.txt From c8111b038029d713fa6535340e12d8bfe17ed868 Mon Sep 17 00:00:00 2001 From: Norbert Date: Fri, 30 Jun 2023 09:28:11 +0200 Subject: [PATCH 4/8] test is done --- get_recommended_whitelist.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/get_recommended_whitelist.sh b/get_recommended_whitelist.sh index 4d2ede4..74c4c92 100644 --- a/get_recommended_whitelist.sh +++ b/get_recommended_whitelist.sh @@ -7,7 +7,6 @@ touch whitelist.csv # declare an array of urls urls=( - https://adaway.org/hosts.txt #test https://raw.githubusercontent.com/im-sm/Pi-hole-Torrent-Blocklist/main/all-torrent-trackres.txt https://raw.githubusercontent.com/AdguardTeam/HttpsExclusions/master/exclusions/banks.txt https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist.txt From c53b735e7af78df46f68075dc57716f0c98b95d9 Mon Sep 17 00:00:00 2001 From: Norbert Date: Fri, 30 Jun 2023 10:46:03 +0200 Subject: [PATCH 5/8] ping for healtcheck --- README.md | 4 ++++ auto_update_github_action.yml | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/README.md b/README.md index 43afebb..5fb8ed5 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ Cloudflare Gateway allows you to create custom rules to filter HTTP, DNS, and ne - Automatically cleans up filter lists: removes duplicates, invalid domains, comments and more - Works fully unattended - Whitelist support, allowing you to prevent false positives and breakage by forcing trusted domains to always be unblocked. +- Optional health check: Sends a ping request ensuring continuous monitoring and alerting for the workflow execution. + ## Usage @@ -53,6 +55,8 @@ Please note that the GitHub Action downloads the recommended blocklists and whit - `CLOUDFLARE_ACCOUNT_ID`: Your Cloudflare account ID - `CLOUDFLARE_ACCOUNT_EMAIL`: Your Cloudflare account email - `CLOUDFLARE_LIST_ITEM_LIMIT`: The maximum number of blocked domains allowed for your Cloudflare Zero Trust plan. Use 300000 for the free plan or if you're unsure. +- `PING_URL`: /Optional/ Your https ping URL + 3. Create a new file in the repository named `.github/workflows/main.yml` with the contents of `auto_update_github_action.yml` found in this repository. The default settings will update your filters every week at 3 AM UTC. You can change this by editing the `schedule` property. 4. Enable GitHub Actions in your repository settings. diff --git a/auto_update_github_action.yml b/auto_update_github_action.yml index 747b937..41dadfe 100644 --- a/auto_update_github_action.yml +++ b/auto_update_github_action.yml @@ -56,3 +56,8 @@ jobs: CLOUDFLARE_ACCOUNT_EMAIL: ${{ secrets.CLOUDFLARE_ACCOUNT_EMAIL }} CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} CLOUDFLARE_LIST_ITEM_LIMIT: ${{ secrets.CLOUDFLARE_LIST_ITEM_LIMIT }} + + - name: Send ping request + if: env.PING_URL != '' + run: | + node -e "var https=require('https');https.get('${{ secrets.PING_URL }}').on('error',err=>console.log('Ping failed: '+err));" From b697683828c4ec631542fee053cdfdf0344228b9 Mon Sep 17 00:00:00 2001 From: Norbert Date: Fri, 30 Jun 2023 11:13:31 +0200 Subject: [PATCH 6/8] Send ping request fixup --- auto_update_github_action.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/auto_update_github_action.yml b/auto_update_github_action.yml index 41dadfe..5958d00 100644 --- a/auto_update_github_action.yml +++ b/auto_update_github_action.yml @@ -59,5 +59,8 @@ jobs: - name: Send ping request if: env.PING_URL != '' + working-directory: cloudflare-gateway-pihole-scripts + env: + PING_URL: ${{ secrets.PING_URL }} run: | - node -e "var https=require('https');https.get('${{ secrets.PING_URL }}').on('error',err=>console.log('Ping failed: '+err));" + node -e "var https=require('https');https.get('${{ env.PING_URL }}').on('error',err=>console.log('Ping failed: '+err));" From 780bae6cad56ea814356fe7df018ad895b434194 Mon Sep 17 00:00:00 2001 From: Norbert Joni Date: Fri, 30 Jun 2023 20:58:02 +0200 Subject: [PATCH 7/8] Update auto_update_github_action.yml Co-authored-by: mrrfv --- auto_update_github_action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_update_github_action.yml b/auto_update_github_action.yml index 5958d00..c86cc77 100644 --- a/auto_update_github_action.yml +++ b/auto_update_github_action.yml @@ -63,4 +63,4 @@ jobs: env: PING_URL: ${{ secrets.PING_URL }} run: | - node -e "var https=require('https');https.get('${{ env.PING_URL }}').on('error',err=>console.log('Ping failed: '+err));" + curl "${{ env.PING_URL }}" From 1648f5444dedb30ea98c22c0869aa25ab06fef30 Mon Sep 17 00:00:00 2001 From: Norbert Joni Date: Fri, 30 Jun 2023 20:58:31 +0200 Subject: [PATCH 8/8] Update README.md Co-authored-by: mrrfv --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5fb8ed5..a5d4ec8 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Please note that the GitHub Action downloads the recommended blocklists and whit - `CLOUDFLARE_ACCOUNT_ID`: Your Cloudflare account ID - `CLOUDFLARE_ACCOUNT_EMAIL`: Your Cloudflare account email - `CLOUDFLARE_LIST_ITEM_LIMIT`: The maximum number of blocked domains allowed for your Cloudflare Zero Trust plan. Use 300000 for the free plan or if you're unsure. -- `PING_URL`: /Optional/ Your https ping URL +- `PING_URL`: /Optional/ The HTTP(S) URL to ping (using curl) after the GitHub Action has successfully updated your filters. Useful for monitoring. 3. Create a new file in the repository named `.github/workflows/main.yml` with the contents of `auto_update_github_action.yml` found in this repository. The default settings will update your filters every week at 3 AM UTC. You can change this by editing the `schedule` property.