Update README.md

.github/workflows/run.yml

@@ -15,9 +15,7 @@ jobs:
       - name: Set variables
         run: |
           echo "GOAMD64=v3" >> $GITHUB_ENV
-          echo "RELEASE_NAME=Released on $(date +%Y%m%d%H%M)" >> $GITHUB_ENV
+          echo "BUILDTIME=$(TZ=Asia/Shanghai date +'%Y-%m-%d %H:%M')" >> $GITHUB_ENV
-          echo "BUILDTIME=$(TZ=Asia/Shanghai date +"%A %B %d, %Y %H:%M")" >> $GITHUB_ENV
-          echo "BUILDTIMECN=$(TZ=Asia/Shanghai date +"%Y年%m月%d日 %H时%M分")" >> $GITHUB_ENV
           echo "CHINA_DOMAINS_URL=https://raw.githubusercontent.com/blackmatrix7/ios_rule_script/master/rule/Clash/ChinaMax/ChinaMax_Domain.txt" >> $GITHUB_ENV
           echo "GOOGLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/google.china.conf" >> $GITHUB_ENV
           echo "APPLE_DOMAINS_URL=https://raw.githubusercontent.com/felixonmars/dnsmasq-china-list/master/apple.china.conf" >> $GITHUB_ENV
@@ -28,41 +26,41 @@ jobs:
           echo "WIN_EXTRA=https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/extra.txt" >> $GITHUB_ENV
         shell: bash
 
-      - name: Checkout the "hidden" branch
+      - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: MetaCubeX/meta-rules-dat
-          ref: hidden
 
       - name: Checkout Loyalsoldier/domain-list-custom
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: Loyalsoldier/domain-list-custom
           path: custom
 
       - name: Checkout v2fly/domain-list-community
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: v2fly/domain-list-community
           path: community
 
+      - name: Checkout MetaCubeX/meta-rules-converter
+        uses: actions/checkout@v4
+        with:
+          repository: MetaCubeX/meta-rules-converter
+          path: convert
+
       - name: Checkout cokebar/gfwlist2dnsmasq
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: cokebar/gfwlist2dnsmasq
           path: gfwlist2dnsmasq
 
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version-file: ./custom/go.mod
+          go-version: '1.22'
           cache-dependency-path: ./custom/go.sum
 
-      - name: Get geoip.dat relative files
-        run: |
-          wget https://github.com/Loyalsoldier/geoip/raw/release/geoip.dat
-          wget https://github.com/Loyalsoldier/geoip/raw/release/geoip.dat.sha256sum
-
       - name: Generate GFWList domains
         run: |
           cd gfwlist2dnsmasq || exit 1
@@ -83,13 +81,13 @@ jobs:
 
       - name: Reserve `full`, `regexp` and `keyword` type of rules from custom lists to "reserve" files
         run: |
-          # curl -sSL ${CUSTOM_DIRECT} | grep -v google | grep -v ooklaserver | grep -v "acg.rip" | perl -ne '/^((full|regexp|keyword):[^:]+)(\n$|:@.+)/ && print "$1\n"' | sort --ignore-case -u > direct-reserve.txt
+          # curl -sSL ${CUSTOM_DIRECT} | grep -v google | grep -v manhua | grep -v ooklaserver | grep -v "acg.rip" | perl -ne '/^((full|regexp|keyword):[^:]+)(\n$|:@.+)/ && print "$1\n"' | sort --ignore-case -u > direct-reserve.txt
           curl -sSL ${CUSTOM_PROXY} | grep -Ev ":@cn" | perl -ne '/^((full|regexp|keyword):[^:]+)(\n$|:@.+)/ && print "$1\n"' | sort --ignore-case -u > proxy-reserve.txt
           
       - name: Add proxy, direct and reject domains from "hidden" branch to appropriate temp files
         run: |
-          cat proxy.txt >> temp-proxy.txt
+          cat ./resouces/proxy.txt >> temp-proxy.txt
-          cat direct.txt >> temp-direct.txt
+          cat ./resouces/direct.txt >> temp-direct.txt
           # cat reject.txt >> temp-reject.txt
 
       - name: Sort and generate redundant lists
@@ -100,24 +98,24 @@ jobs:
 
       - name: Remove redundant domains
         run: |
-          chmod +x findRedundantDomain.py
+          chmod +x ./resouces/*.py
-          ./findRedundantDomain.py ./direct-list-with-redundant ./direct-list-deleted-unsort
+          python ./resouces/findRedundantDomain.py ./direct-list-with-redundant ./direct-list-deleted-unsort
-          ./findRedundantDomain.py ./proxy-list-with-redundant ./proxy-list-deleted-unsort
+          python ./resouces/findRedundantDomain.py ./proxy-list-with-redundant ./proxy-list-deleted-unsort
           [ ! -f "direct-list-deleted-unsort" ] && touch direct-list-deleted-unsort
           [ ! -f "proxy-list-deleted-unsort" ] && touch proxy-list-deleted-unsort
           sort ./direct-list-deleted-unsort > ./direct-list-deleted-sort
           sort ./proxy-list-deleted-unsort > ./proxy-list-deleted-sort
-          diff ./direct-list-deleted-sort ./direct-list-with-redundant | awk '/^>/{print $2}' > ./direct-list-without-redundant
+          python ./resouces/removeFrom.py -remove ./direct-list-deleted-sort -from ./direct-list-with-redundant -out direct-list-without-redundant
-          diff ./proxy-list-deleted-sort ./proxy-list-with-redundant | awk '/^>/{print $2}' > ./proxy-list-without-redundant
+          python ./resouces/removeFrom.py -remove ./proxy-list-deleted-sort -from ./proxy-list-with-redundant -out proxy-list-without-redundant
 
       - name: Remove domains from "need-to-remove" lists in "hidden" branch
         run: |
-          diff ./direct-need-to-remove.txt ./direct-list-without-redundant | awk '/^>/{print $2}' > temp-cn.txt
+          python ./resouces/removeFrom.py -remove ./resouces/direct-need-to-remove.txt -from ./direct-list-without-redundant -out ./temp-cn.txt
-          diff ./proxy-need-to-remove.txt ./proxy-list-without-redundant | awk '/^>/{print $2}' > temp-geolocation-\!cn.txt
+          python ./resouces/removeFrom.py -remove ./resouces/proxy-need-to-remove.txt -from ./proxy-list-without-redundant -out ./temp-geolocation-\!cn.txt
 
       - name: Remove domains end with ".cn" in "temp-geolocation-!cn.txt" and write lists to data directory
         run: |
-          cat temp-cn.txt | grep -v google  | grep -v ooklaserver | grep -v "acg.rip" | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' > ./community/data/cn
+          cat temp-cn.txt | grep -v google | grep -v manhua | grep -v ooklaserver | grep -v "acg.rip" | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' > ./community/data/cn
           cat temp-cn.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > direct-tld-list.txt
           cat temp-geolocation-\!cn.txt | sort --ignore-case -u | perl -ne '/^((?=^.{1,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})*)/ && print "$1\n"' | perl -ne 'print if not /\.cn$/' > ./community/data/geolocation-\!cn
           cat temp-geolocation-\!cn.txt | sort --ignore-case -u | perl -ne 'print if not /^((?=^.{3,255})[a-zA-Z0-9][-_a-zA-Z0-9]{0,62}(\.[a-zA-Z0-9][-_a-zA-Z0-9]{0,62})+)/' > proxy-tld-list.txt
@@ -197,10 +195,12 @@ jobs:
           curl -sSL https://github.com/blackmatrix7/ios_rule_script/raw/master/rule/Clash/Twitter/Twitter.yaml | ${{ env.SED }} > ./data-lite/twitter
           curl -sSL https://github.com/blackmatrix7/ios_rule_script/raw/master/rule/Clash/TikTok/TikTok.yaml | ${{ env.SED }} > ./data-lite/tiktok
           curl -sSL https://github.com/blackmatrix7/ios_rule_script/raw/master/rule/Clash/YouTube/YouTube.yaml | ${{ env.SED }} > ./data-lite/youtube
+          curl -sSL https://github.com/blackmatrix7/ios_rule_script/raw/master/rule/Clash/GlobalMedia/GlobalMedia.yaml | ${{ env.SED }} > ./data-lite/proxymedia
 
           echo include:google >> ./data-lite/proxy && echo include:github >> ./data-lite/proxy && echo include:netflix >> ./data-lite/proxy
           echo ipleak.net >> ./data-lite/proxy && echo browserleaks.org >> ./data-lite/proxy
           echo "full:o33249.ingest.sentry.io" >>  ./data-lite/openai
+          echo "openai.com" >> ./data-lite/openai
           go run ./ --datapath=./data-lite --outputname geosite-lite.dat
 
       - name: Get geoip.dat relative files
@@ -220,11 +220,70 @@ jobs:
           geo convert ip -i v2ray -o sing -f geoip-lite.db ./geoip-lite.dat
           geo convert ip -i v2ray -o meta -f geoip-lite.metadb ./geoip-lite.dat
 
+      - name: Convert geo to sing-rule-set
+        env:
+          NO_SKIP: true
+        run: |
+          mkdir -p sing-rule/geo/geosite && mkdir -p sing-rule/geo/geoip
+          cd convert
+          go run ./ geosite -f ../custom/publish/geosite.dat -o ../sing-rule/geo/geosite -t sing-box
+          go run ./ geoip -f ../geoip.dat -o ../sing-rule/geo/geoip -t sing-box
+
+      - name: Convert geo-lite to sing-rule-set
+        env:
+          NO_SKIP: true
+        run: |
+          mkdir -p sing-rule/geo-lite/geosite && mkdir -p sing-rule/geo-lite/geoip
+          cd convert
+          go run ./ geosite -f ../community/geosite-lite.dat -o ../sing-rule/geo-lite/geosite -t sing-box
+          go run ./ geoip -f ../geoip-lite.dat -o ../sing-rule/geo-lite/geoip -t sing-box
+
+      - name: Convert geo to meta-rule-set
+        env:
+          NO_SKIP: true
+        run: |
+          mkdir -p meta-rule/geo/geosite && mkdir -p meta-rule/geo/geoip
+          cd convert
+          go run ./ geosite -f ../custom/publish/geosite.dat -o ../meta-rule/geo/geosite
+          go run ./ geoip -f ../geoip.dat -o ../meta-rule/geo/geoip
+
+      - name: Convert geo-lite to meta-rule-set
+        env:
+          NO_SKIP: true
+        run: |
+          mkdir -p meta-rule/geo-lite/geosite && mkdir -p meta-rule/geo-lite/geoip
+          cd convert
+          go run ./ geosite -f ../community/geosite-lite.dat -o ../meta-rule/geo-lite/geosite
+          go run ./ geoip -f ../geoip-lite.dat -o ../meta-rule/geo-lite/geoip
+
+      - name: Convert asn
+        env:
+          NO_SKIP: true
+        run: |
+          mkdir -p meta-rule/asn && mkdir -p sing-rule/asn
+          cd convert
+          wget https://raw.githubusercontent.com/xishang0128/geoip/release/GeoLite2-ASN.mmdb
+          go run ./ asn -o ../meta-rule/asn
+          go run ./ asn -o ../sing-rule/asn -t sing-box
+
+      # - name: Convert bm7-rule-set to sing-rule-set
+      #   env:
+      #     NO_SKIP: true
+      #   run: |
+      #     mkdir -p ./sing-rule/bm7
+      #     cp ./resouces/bm7.sh ./sing-rule/bm7/
+      #     cd ./sing-rule/bm7
+      #     wget https://github.com/SagerNet/sing-box/releases/download/v1.8.0-alpha.1/sing-box-1.8.0-alpha.1-linux-amd64.tar.gz -O sing-box.tar.gz && tar zxvf sing-box.tar.gz && mv sing-box-1.8.0-alpha.1-linux-amd64/sing-box ./ && rm -r sing-box-1.8.0-alpha.1-linux-amd64
+      #     ls
+      #     chmod 755 bm7.sh && ./bm7.sh
+      #     rm -r bm7.sh sing-box* rule
+
       - name: Move and zip files
         run: |
           mkdir -p ./publish/
           wget https://raw.githubusercontent.com/xishang0128/geoip/release/Country.mmdb -O ./publish/country-lite.mmdb
           wget https://raw.githubusercontent.com/Loyalsoldier/geoip/release/Country.mmdb -O ./publish/country.mmdb
+          wget https://raw.githubusercontent.com/xishang0128/geoip/release/GeoLite2-ASN.mmdb -O ./publish/GeoLite2-ASN.mmdb
           install -Dp ./geoip-lite.dat ./publish/
           install -Dp ./geoip.dat ./publish/
           install -Dp ./custom/publish/geosite.dat ./publish/
@@ -235,10 +294,9 @@ jobs:
           install -Dp ./geoip-lite.db ./publish/
           install -Dp ./geoip.metadb ./publish/
           install -Dp ./geoip-lite.metadb ./publish/
-          curl -sSL https://raw.githubusercontent.com/blackmatrix7/ios_rule_script/master/rule/Clash/ChinaMax/ChinaMax_Domain.yaml | sed '/^\s*#/d' > ./publish/cn_domain.yaml
-          curl -sSL https://raw.githubusercontent.com/blackmatrix7/ios_rule_script/master/rule/Clash/Proxy/Proxy_Domain.yaml | sed '/^\s*#/d' > ./publish/proxy.yaml
-          curl -sSL https://raw.githubusercontent.com/blackmatrix7/ios_rule_script/master/rule/Clash/ChinaMax/ChinaMax_Classical.yaml | grep DOMAIN | grep -v "#" | sed 's/  - DOMAIN,/full:/g' | sed 's/  - DOMAIN-SUFFIX,//g' | sed 's/  - DOMAIN-KEYWORD,/keyword:/g' > ./publish/cn.txt
           cd ./publish || exit 1
+          sha256sum country.mmdb > country.mmdb.sha256sum
+          sha256sum country-lite.mmdb > country-lite.mmdb.sha256sum
           sha256sum geoip-lite.dat > geoip-lite.dat.sha256sum
           sha256sum geoip.dat > geoip.dat.sha256sum
           sha256sum geosite.dat > geosite.dat.sha256sum
@@ -249,6 +307,7 @@ jobs:
           sha256sum geoip-lite.db > geoip-lite.db.sha256sum
           sha256sum geoip.metadb > geoip.metadb.sha256sum
           sha256sum geoip-lite.metadb > geoip-lite.metadb.sha256sum
+          sha256sum GeoLite2-ASN.mmdb > GeoLite2-ASN.mmdb.sha256sum
 
       - name: Delete current release assets
         uses: andreaswilli/delete-release-assets-action@v3.0.0
@@ -257,19 +316,6 @@ jobs:
           tag: latest
           deleteOnlyFromDrafts: false
 
-      - name: Create Release Body
-        id: create_release_body
-        run: |
-          cat > release.txt << 'EOF'
-          Release update at  ${{ env.BUILDTIME }}
-          更新时间 ${{ env.BUILDTIMECN }}
-          EOF
-          r=$(cat release.txt)
-          r="${r//'%'/'%25'}"
-          r="${r//$'\n'/'%0A'}"
-          r="${r//$'\r'/'%0D'}"
-          echo "RELEASE_BODY=$r" >> $GITHUB_ENV
-
       - name: Create and Upload Release
         id: upload_release
         uses: svenstaro/upload-release-action@v2
@@ -278,8 +324,8 @@ jobs:
           release_name: Release ${{ env.BUILDTIME }}
           tag: latest
           file_glob: true
+          overwrite: true
           file: ./publish/*
-          body: ${{ env.RELEASE_BODY }}
 
       - name: Git push assets to "release" branch
         run: |
@@ -289,10 +335,35 @@ jobs:
           git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git checkout -b release
           git add .
-          git commit -m "${{ env.RELEASE_NAME }}"
+          git commit -m "Released on ${{ env.BUILDTIME }}"
           git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}"
           git push -f -u origin release
 
+      - name: Git push assets to "sing-rule" branch
+        run: |
+            cd sing-rule || exit 1
+            git init
+            git config --local user.name "github-actions[bot]"
+            git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+            git checkout -b sing
+            git add .
+            git commit -m "Released on ${{ env.BUILDTIME }}"
+            git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}"
+            git push -f -u origin sing
+
+      - name: Git push assets to "meta-rule" branch
+        run: |
+            cd meta-rule || exit 1
+            git init
+            git config --local user.name "github-actions[bot]"
+            git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+            git checkout -b meta
+            git add .
+            git commit -m "Released on ${{ env.BUILDTIME }}"
+            git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}"
+            git push -f -u origin meta
+
+
       - name: Purge jsdelivr CDN
         run: |
           cd publish || exit 1

README.md

@@ -1,80 +1,30 @@
 ## **下载地址**：
 
-### geoX
+| 文件名              | Github release                                                                                                            | JSdelivr                                                                                                                           | JSdelivr-CF                                                                                                                              |
-- **country.mmdb**
+|---------------------|---------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------|
-  - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/country.mmdb)
+| country.mmdb        | [下载](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/country.mmdb)                                 | [下载](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/country.mmdb)                                                  | [下载](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/country.mmdb)                                                  |
-  - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/country.mmdb)
+| geoip.dat           | [下载](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geoip.dat)                                    | [下载](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.dat)                                                       | [下载](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.dat)                                                       |
-  - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/country.mmdb)
+| geoip.db            | [下载](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geoip.db)                                     | [下载](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.db)                                                        | [下载](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.db)                                                        |
-  
+| geoip.metadb        | [下载](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geoip.metadb)                                 | [下载](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.metadb)                                                    | [下载](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.metadb)                                                    |
-- **geoip.dat**
+| country-lite.mmdb   | [下载](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/country-lite.mmdb)                            | [下载](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/country-lite.mmdb)                                               | [下载](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/country-lite.mmdb)                                               |
-  - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geoip.dat)
+| geoip-lite.dat      | [下载](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geoip-lite.dat)                               | [下载](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip-lite.dat)                                                  | [下载](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip-lite.dat)                                                  |
-  - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.dat)
+| geoip-lite.db       | [下载](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geoip-lite.db)                                | [下载](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip-lite.db)                                                   | [下载](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip-lite.db)                                                   |
-  - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.dat)
+| geosite.dat         | [下载](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geosite.dat)                                  | [下载](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite.dat)                                                     | [下载](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite.dat)                                                     |
-
+| geosite.db          | [下载](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geosite.db)                                   | [下载](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite.db)                                                      | [下载](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite.db)                                                      |
-- **geoip.db**
+| geosite-lite.dat    | [下载](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geosite-lite.dat)                             | [下载](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite-lite.dat)                                                 | [下载](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite-lite.dat)                                                 |
-  - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geoip.db)
+| geosite-lite.db     | [下载](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geosite-lite.db)                              | [下载](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite-lite.db)                                                 | [下载](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite-lite.db)                                                 |
-  - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.db)
+| GeoLite2-ASN.mmdb   | [下载](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/GeoLite2-ASN.mmdb)                              | [下载](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/GeoLite2-ASN.mmdb)                                                 | [下载](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/GeoLite2-ASN.mmdb)                                                 |
-  - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.db)
-
-- **geoip.metadb**
-  - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geoip.metadb)
-  - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.metadb)
-  - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip.metadb)
-
-- **country-lite.mmdb**
-  - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/country-lite.mmdb)
-  - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/country-lite.mmdb)
-  - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/country-lite.mmdb)
-  
-- **geoip-lite.dat**
-  - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geoip-lite.dat)
-  - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip-lite.dat)
-  - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip-lite.dat)
-
-- **geoip-lite.db**
-  - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geoip-lite.db)
-  - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip-lite.db)
-  - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip-lite.db)
-  - 
-- **geoip-lite.mmdb**
-  - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geoip-lite.mmdb)
-  - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip-lite.mmdb)
-  - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geoip-lite.mmdb)
-  
-- **geosite.dat**
-  - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geosite.dat)
-  - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite.dat)
-  - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite.dat)
-
-- **geosite.db**
-  - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geosite.db)
-  - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite.db)
-  - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite.db)
-
-- **geosite-lite.dat**
-  - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geosite-lite.dat)
-  - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite-lite.dat)
-  - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite-lite.dat)
-
-- **geosite-lite.db**
-  - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/geosite-lite.db)
-  - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite-lite.db)
-  - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/geosite-lite.db)
 
 ### **rule-set**
-  - **cn_domain.yaml**
-    - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/cn_domain.yaml)
-    - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/cn_domain.yaml)
-    - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/cn_domain.yaml)
 
-  - **proxy.yaml**
+mihomo：[meta branch](https://github.com/MetaCubeX/meta-rules-dat/tree/meta)
-    - [Github release](https://github.com/MetaCubeX/meta-rules-dat/releases/download/latest/proxy.yaml)
+
-    - [JSdelivr](https://cdn.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/proxy.yaml)
+sing-box: [sing branch](https://github.com/MetaCubeX/meta-rules-dat/tree/sing)
-    - [JSdelivr-CF](https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/proxy.yaml)
 
 ## **country.mmdb,geoip.dat,geoip.db 内容**
 
 同 [Loyalsoldier/v2ray-rules-dat](https://github.com/Loyalsoldier/v2ray-rules-dat)
+
 - 新增类别（方便有特殊需求的用户使用）：
   - `geoip:cloudflare`
   - `geoip:cloudfront`
@@ -88,6 +38,7 @@
 ## **country-lite.mmdb,geoip-lite.dat,geoip-lite.db 内容**
 
 国家仅包含 CN/JP,精简体积,替换一些类别
+
 - 新增类别（方便有特殊需求的用户使用）：
   - `geoip:cloudflare`
   - `geoip:cloudfront`
@@ -102,26 +53,24 @@
 ## **geosite.dat,geosite.db 内容**
 
 用法同 [Loyalsoldier/v2ray-rules-dat](https://github.com/Loyalsoldier/v2ray-rules-dat)
+
 - `geosite:category-ads-all` 仅使用域名作为广告拦截用途作用有限，因此不作额外域名添加
 - `geosite:cn` 源替换为 [ios_rule_script/ChinaMax_Domain](https://github.com/blackmatrix7/ios_rule_script/tree/master/rule/Clash/ChinaMax)
 - `geosite:onedrive` 合并 [ios_rule_script/OneDrive](https://github.com/blackmatrix7/ios_rule_script/tree/master/rule/Clash/OneDrive)
 - `geosite:steam@cn` 合并 [ios_rule_script/SteamCN](https://github.com/blackmatrix7/ios_rule_script/tree/master/rule/Clash/SteamCN) 的内数据
-  - 新增类别
+- 新增类别 - `geosite:biliintl` 来源 [biliintl](https://raw.githubusercontent.com/xishang0128/rules/main/biliintl.list) - `geosite:tracker` 来源 [TrackersList](https://trackerslist.com/#/zh)以及[blackmatrix7
-    - `geosite:biliintl` 来源 [biliintl](https://raw.githubusercontent.com/xishang0128/rules/main/biliintl.list)
-    - `geosite:tracker` 来源 [TrackersList](https://trackerslist.com/#/zh)以及[blackmatrix7
   /
   ios_rule_script](https://github.com/blackmatrix7/ios_rule_script/tree/master/rule/Clash/PrivateTracker)
 
-
 ## **geosite-lite.dat,geosite-lite.db 内容**
 
-仅包含常用集合,cn为精简集合,可能不全
+仅包含常用集合，cn 为精简集合，可能不全
 集合内容均来自 https://github.com/blackmatrix7/ios_rule_script/tree/master/rule/Clash
 
-集合包含 `abema / apple / applemusic / bilibili / biliintl / bahamut / cn / ehentai / google / github / microsoft / netflix / openai / onedrive / pixiv / proxy / spotify / telegram / twitter / tiktok / youtube`
+集合包含 `abema / apple / applemusic / bilibili / biliintl / bahamut / cn / ehentai / google / github / microsoft / netflix / openai / onedrive / pixiv / proxy / spotify / telegram / twitter / tiktok / youtube / proxymedia`
-
 
 ## **示例**
+
 ```yaml
 rule-providers:
   cn:
@@ -129,7 +78,7 @@ rule-providers:
     interval: 86400
     path: ./provider/rule-set/cn_domain.yaml
     type: http
-    url: "https://testingcf.jsdelivr.net/gh/MetaCubeX/meta-rules-dat@release/cn_domain.yaml"
+    url: "https://raw.githubusercontent.com/MetaCubeX/meta-rules-dat/meta/geo/geosite/cn.yaml"
 
 rules:
   - RULE-SET,cn,DIRECT
@@ -158,6 +107,12 @@ rules:
   - MATCH,DIRECT
 ```
 
+## 辅助工具
+
+https://github.com/MetaCubeX/geo
+
+🗺 An easy way to manage all your Geo resources.
+
 ## 致谢
 
 - [@Loyalsoldier/geoip](https://github.com/Loyalsoldier/geoip)

resouces/bm7.sh

@@ -0,0 +1,134 @@
+#!/bin/bash
+
+# 拉文件
+if [ ! -d rule ]; then
+	git init
+	git remote add origin https://github.com/blackmatrix7/ios_rule_script.git
+	git config core.sparsecheckout true
+	echo "rule/Clash" >>.git/info/sparse-checkout
+	git pull --depth 1 origin master
+	rm -rf .git
+fi
+# 移动文件/目录到同一文件夹
+list=($(find ./rule/Clash/ | awk -F '/' '{print $5}' | sed '/^$/d' | grep -v '\.' | sort -u))
+for ((i = 0; i < ${#list[@]}; i++)); do
+	path=$(find ./rule/Clash/ -name ${list[i]})
+	mv $path ./rule/Clash/
+done
+
+list=($(ls ./rule/Clash/))
+for ((i = 0; i < ${#list[@]}; i++)); do
+	if [ -z "$(ls ./rule/Clash/${list[i]} | grep '.yaml')" ]; then
+		directory=($(ls ./rule/Clash/${list[i]}))
+		for ((x = 0; x < ${#directory[@]}; x++)); do
+			mv ./rule/Clash/${list[i]}/${directory[x]} ./rule/Clash/${directory[x]}
+		done
+		rm -r ./rule/Clash/${list[i]}
+	fi
+done
+
+list=($(ls ./rule/Clash/))
+for ((i = 0; i < ${#list[@]}; i++)); do
+	if [ -f "./rule/Clash/${list[i]}/${list[i]}_Classical.yaml" ]; then
+		mv ./rule/Clash/${list[i]}/${list[i]}_Classical.yaml ./rule/Clash/${list[i]}/${list[i]}.yaml
+	fi
+done
+
+# 处理文件
+list=($(ls ./rule/Clash/))
+for ((i = 0; i < ${#list[@]}; i++)); do
+	mkdir -p ${list[i]}
+	# 归类
+	# android package
+	if [ -n "$(cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep -v '#' | grep PROCESS | grep -v '\.exe' | grep -v '/' | grep '\.')" ]; then
+		cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep -v '#' |  grep PROCESS | grep -v '\.exe' | grep -v '/' | grep '\.' | sed 's/  - PROCESS-NAME,//g' > ${list[i]}/package.json
+	fi
+	# process name
+	if [ -n "$(cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep -v '#' | grep PROCESS | grep -v '/' | grep -v '\.')" ]; then
+		cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep -v '#' | grep -v '#' | grep PROCESS | grep -v '/' | grep -v '\.' | sed 's/  - PROCESS-NAME,//g' > ${list[i]}/process.json
+	fi
+	if [ -n "$(cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep -v '#' | grep PROCESS |  grep '\.exe')" ]; then
+		cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep -v '#' | grep -v '#' | grep PROCESS |  grep '\.exe' | sed 's/  - PROCESS-NAME,//g' >> ${list[i]}/process.json
+	fi
+	# domain
+	if [ -n "$(cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep '\- DOMAIN-SUFFIX,')" ]; then
+		cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep -v '#' | grep '\- DOMAIN-SUFFIX,' | sed 's/  - DOMAIN-SUFFIX,//g' > ${list[i]}/domain.json
+		cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep -v '#' | grep '\- DOMAIN-SUFFIX,' | sed 's/  - DOMAIN-SUFFIX,/./g' > ${list[i]}/suffix.json
+	fi
+	if [ -n "$(cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep '\- DOMAIN,')" ]; then
+		cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep -v '#' | grep '\- DOMAIN,' | sed 's/  - DOMAIN,//g' >> ${list[i]}/domain.json
+	fi
+	if [ -n "$(cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep '\- DOMAIN-KEYWORD,')" ]; then
+		cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep -v '#' | grep '\- DOMAIN-KEYWORD,' | sed 's/  - DOMAIN-KEYWORD,//g' > ${list[i]}/keyword.json
+	fi
+	# ipcidr
+	if [ -n "$(cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep '\- IP-CIDR')" ]; then
+		cat ./rule/Clash/${list[i]}/${list[i]}.yaml | grep -v '#' | grep '\- IP-CIDR' | sed 's/  - IP-CIDR,//g' | sed 's/  - IP-CIDR6,//g' > ${list[i]}/ipcidr.json
+	fi
+	# 转成json格式
+	# android package
+	if [ -f "${list[i]}/package.json" ]; then
+		sed -i 's/^/        "/g' ${list[i]}/package.json
+		sed -i 's/$/",/g' ${list[i]}/package.json
+		sed -i '1s/^/      "package_name": [\n/g' ${list[i]}/package.json
+		sed -i '$ s/,$/\n      ],/g' ${list[i]}/package.json
+	fi
+	# process name
+	if [ -f "${list[i]}/process.json" ]; then
+		sed -i 's/^/        "/g' ${list[i]}/process.json
+		sed -i 's/$/",/g' ${list[i]}/process.json
+		sed -i '1s/^/      "process_name": [\n/g' ${list[i]}/process.json
+		sed -i '$ s/,$/\n      ],/g' ${list[i]}/process.json
+	fi
+	# domain
+	if [ -f "${list[i]}/domain.json" ]; then
+		sed -i 's/^/        "/g' ${list[i]}/domain.json
+		sed -i 's/$/",/g' ${list[i]}/domain.json
+		sed -i '1s/^/      "domain": [\n/g' ${list[i]}/domain.json
+		sed -i '$ s/,$/\n      ],/g' ${list[i]}/domain.json
+	fi
+	if [ -f "${list[i]}/suffix.json" ]; then
+		sed -i 's/^/        "/g' ${list[i]}/suffix.json
+		sed -i 's/$/",/g' ${list[i]}/suffix.json
+		sed -i '1s/^/      "domain_suffix": [\n/g' ${list[i]}/suffix.json
+		sed -i '$ s/,$/\n      ],/g' ${list[i]}/suffix.json
+	fi
+	if [ -f "${list[i]}/keyword.json" ]; then
+		sed -i 's/^/        "/g' ${list[i]}/keyword.json
+		sed -i 's/$/",/g' ${list[i]}/keyword.json
+		sed -i '1s/^/      "domain_keyword": [\n/g' ${list[i]}/keyword.json
+		sed -i '$ s/,$/\n      ],/g' ${list[i]}/keyword.json
+	fi
+	# ipcidr
+	if [ -f "${list[i]}/ipcidr.json" ]; then
+		sed -i 's/^/        "/g' ${list[i]}/ipcidr.json
+		sed -i 's/$/",/g' ${list[i]}/ipcidr.json
+		sed -i '1s/^/      "ip_cidr": [\n/g' ${list[i]}/ipcidr.json
+		sed -i '$ s/,$/\n      ],/g' ${list[i]}/ipcidr.json
+	fi
+	# 合并文件
+	if [ -f "${list[i]}/package.json" -a -f "${list[i]}/process.json" ]; then
+		mv ${list[i]}/package.json ${list[i]}.json
+		sed -i '$ s/,$/\n    },\n    {/g' ${list[i]}.json
+		cat ${list[i]}/process.json >> ${list[i]}.json
+		rm ${list[i]}/process.json
+	elif [ -f "${list[i]}/package.json" ]; then
+		mv ${list[i]}/package.json ${list[i]}.json
+	elif [ -f "${list[i]}/process.json" ]; then
+		mv ${list[i]}/process.json ${list[i]}.json
+	fi
+
+	if [ "$(ls ${list[i]})" = "" ]; then
+		sed -i '1s/^/{\n  "version": 1,\n  "rules": [\n    {\n/g' ${list[i]}.json
+	elif [ -f "${list[i]}.json" ]; then
+		sed -i '1s/^/{\n  "version": 1,\n  "rules": [\n    {\n/g' ${list[i]}.json
+		sed -i '$ s/,$/\n    },\n    {/g' ${list[i]}.json
+		cat ${list[i]}/* >> ${list[i]}.json
+	else
+		cat ${list[i]}/* >> ${list[i]}.json
+		sed -i '1s/^/{\n  "version": 1,\n  "rules": [\n    {\n/g' ${list[i]}.json
+	fi
+	sed -i '$ s/,$/\n    }\n  ]\n}/g' ${list[i]}.json
+	rm -r ${list[i]}
+	./sing-box rule-set compile ${list[i]}.json -o ${list[i]}.srs
+done

resouces/convert-clash.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+mkdir -p geoip
+./mosdns v2dat unpack-ip -o ./geoip/ geoip.dat
+list=($(ls ./geoip | sed 's/geoip_//g' | sed 's/\.txt//g'))
+for ((i = 0; i < ${#list[@]}; i++)); do
+	#	echo "${list[i]}"
+	mv ./geoip/geoip_${list[i]}.txt ./geoip/${list[i]}.list
+	echo "payload:" >./geoip/${list[i]}.yaml
+	cat ./geoip/${list[i]}.list | sed 's/^/- "/g' | sed 's/$/"/g' >>./geoip/${list[i]}.yaml
+done
+
+mkdir -p geosite
+./mosdns v2dat unpack-domain -o ./geosite/ geosite.dat
+list=($(ls ./geosite | sed 's/geosite_//g' | sed 's/\.txt//g'))
+for ((i = 0; i < ${#list[@]}; i++)); do
+	#	echo "${list[i]}"
+	mv ./geosite/geosite_${list[i]}.txt ./geosite/${list[i]}.list
+	sed -i '/^#/d' geosite/${list[i]}.list
+	sed -i '/^keyword:/d' geosite/${list[i]}.list
+	sed -i '/^regexp:/d' geosite/${list[i]}.list
+	sed -i 's/^/+./g' ./geosite/${list[i]}.list
+	sed -i 's/+.full://g' ./geosite/${list[i]}.list
+	sed -i 's/+.domain:/+./g' ./geosite/${list[i]}.list
+	echo "payload:" >./geosite/${list[i]}.yaml
+	cat ./geosite/${list[i]}.list | sed 's/^/- "/g' | sed 's/$/"/g' >>./geosite/${list[i]}.yaml
+done
+
+#mkdir -p mixed
+#for file in $(find geoip -type f | grep -v srs | awk -F "/" '{print $NF}'); do
+#	if [ -n "$(find geosite -type f -iname "$file")" ]; then
+#		file=$(find ./geosite -type f -iname "$file" | awk -F"/" '{print $NF}' | sed 's/\.json//g')
+#		head -n -3 ./geoip/${file}.json >./mixed/${file}.json
+#		sed -i 's/]/],/g' ./mixed/${file}.json
+#		tail -n +5 ./geosite/${file}.json >>./mixed/${file}.json
+#		./sing-box rule-set compile ./mixed/${file}.json -o ./mixed/${file}.srs
+#	fi
+#done

resouces/convert.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+mkdir -p geoip
+./mosdns v2dat unpack-ip -o ./geoip/ geoip.dat
+list=($(ls ./geoip | sed 's/geoip_//g' | sed 's/\.txt//g'))
+for ((i = 0; i < ${#list[@]}; i++)); do
+	sed -i 's/^/        "/g' ./geoip/geoip_${list[i]}.txt
+	sed -i 's/$/",/g' ./geoip/geoip_${list[i]}.txt
+	sed -i '1s/^/{\n  "version": 1,\n  "rules": [\n    {\n      "ip_cidr": [\n/g' ./geoip/geoip_${list[i]}.txt
+	sed -i '$ s/,$/\n      ]\n    }\n  ]\n}/g' ./geoip/geoip_${list[i]}.txt
+	mv ./geoip/geoip_${list[i]}.txt ./geoip/${list[i]}.json
+	./sing-box rule-set compile "./geoip/${list[i]}.json" -o ./geoip/${list[i]}.srs
+done
+
+list=($(./sing-box geosite list | sed 's/ (.*)$//g'))
+mkdir -p geosite
+for ((i = 0; i < ${#list[@]}; i++)); do
+	./sing-box geosite export ${list[i]} -o ./geosite/${list[i]}.json
+	./sing-box rule-set compile ./geosite/${list[i]}.json -o ./geosite/${list[i]}.srs
+done
+
+# mkdir -p mixed
+# for file in $(find geoip -type f | grep -v srs | awk -F "/" '{print $NF}'); do
+# 	if [ -n "$(find geosite -type f -iname "$file")" ]; then
+# 		file=$(find ./geosite -type f -iname "$file" | awk -F"/" '{print $NF}' | sed 's/\.json//g')
+# 		head -n -3 ./geoip/${file}.json >./mixed/${file}.json
+# 		sed -i 's/]/],/g' ./mixed/${file}.json
+# 		tail -n +5 ./geosite/${file}.json >>./mixed/${file}.json
+# 		./sing-box rule-set compile ./mixed/${file}.json -o ./mixed/${file}.srs
+# 	fi
+# done

resouces/direct-need-to-remove.txt

@@ -0,0 +1,36 @@
+103.com
+123cha.com
+95081.com
+acg.rip
+airasia.com
+baid.us
+baidu.jp
+bussou.com
+busytrade.com
+cnbeta.com
+cnbetacdn.com
+cnpolitics.org
+dm530.net
+duanzhihu.com
+dysfz.cc
+emacs-china.org
+galaxymacau.com
+galstars.net
+haitum.com
+hostloc.com
+jiaoyou8.com
+kh.google.com
+laonanren.com
+mysinablog.com
+ntrqq.com
+nytlog.com
+shuangtv.net
+suppig.net
+top
+xclient.info
+xjp.cc
+yanghengjun.com
+ydy.com
+yslang.com
+yysub.net
+hamreus.com

resouces/findRedundantDomain.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+import sys
+print(sys.argv[1], sys.argv[2])
+
+''' Find redundant items in domain lists.
+    e.g. 'bar.foo.com' is redundant for 'foo.com'.
+'''
+
+def load(list):
+    ''' Parse conf file & Prepare data structure
+        Returns: [ ['abc', 'com'],
+                   ['bar', 'foo', 'com'],
+                   ... ]
+    '''
+
+    results = []
+    with open(list, 'r') as f:
+        for line in f.readlines():
+            line = line.strip()
+            if line == '' or line.startswith('#'):
+                continue
+            # A domain name is case-insensitive and
+            # consists of several labels, separated by a full stop
+            domain_labels = line.lower().split('.')
+            results.append(domain_labels)
+
+    # Sort results by domain labels' length
+    results.sort(key=len)
+    return results
+
+def find(labelses, removedDomainFile):
+    ''' Find redundant items by a tree of top-level domain label to sub-level.
+        `tree` is like { 'com': { 'foo: { 'bar': LEAF },
+                                  'abc': LEAF },
+                         'org': ... }
+    '''
+
+    tree = {}
+    LEAF = 1
+    for labels in labelses:
+        domain = '.'.join(labels)
+        # Init root node as current node
+        node = tree
+        while len(labels) > 0:
+            label = labels.pop()
+            if label in node:
+                # If child node is a LEAF node,
+                # current domain must be an existed domain or a subdomain of an existed.
+                if node[label] == LEAF:
+                    print(f"Redundant found: {domain} at {'.'.join(labels)}")
+                    with open(removedDomainFile, "a") as f:
+                        f.write(domain)
+                        f.write("\n")
+                    break
+            else:
+                # Create a leaf node if current label is last one
+                if len(labels) == 0:
+                    node[label] = LEAF
+                # Create a branch node
+                else:
+                    node[label] = {}
+            # Iterate to child node
+            node = node[label]
+
+if __name__ == '__main__':
+    find(load(sys.argv[1]), sys.argv[2])

resouces/proxy-need-to-remove.txt

@@ -0,0 +1,3 @@
+ifanr.com
+weibo.com
+www.baidu.com

resouces/proxy.txt

@@ -0,0 +1,2 @@
+supertop.co
+hk.chinamobile.com

resouces/reject-need-to-remove.txt

@@ -0,0 +1,33 @@
+4paradigm.com
+addthis.com
+addthisedge.com
+alimama.alicdn.com
+alimama.com
+analytics.google.com
+app.chat.xiaomi.net
+bdtj.tagtic.cn
+cdn.onesignal.com
+click.discord.com
+click.redditmail.com
+ctrip.com
+d.ifengimg.com
+icons.mydrivers.com
+img.alibaba.com
+jav321.com
+knet.cn
+mail.tsinghua.edu.cn
+mtalk.google.com
+mx.technolutions.net
+newrelic.com
+offer.alibaba.com
+pingjs.qq.com
+qlogo.cn
+resolver.msg.xiaomi.net
+s.youtube.com
+sf3-ttcdn-tos.pstatp.com
+t.co
+tagtic.cn
+telegra.ph
+tongji.baidu.com
+tv.sohu.com
+ue.yeyoucdn.com

resouces/removeFrom.py

@@ -0,0 +1,29 @@
+import argparse
+
+
+def remove_domains(file_to_remove, file_to_remove_from, output_file):
+    with open(file_to_remove, "r") as f_remove, open(
+        file_to_remove_from, "r"
+    ) as f_from:
+        domains_to_remove = set(line.strip() for line in f_remove)
+        all_domains = set(line.strip() for line in f_from)
+
+    remaining_domains = all_domains - domains_to_remove
+
+    with open(output_file, "w") as output:
+        output.write("\n".join(remaining_domains))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Remove domains from a file.")
+    parser.add_argument(
+        "-remove", required=True, help="File containing domains to be removed"
+    )
+    parser.add_argument(
+        "-from", required=True, dest="from_file", help="File to remove domains from"
+    )
+    parser.add_argument("-out", required=True, help="Output file")
+
+    args = parser.parse_args()
+
+    remove_domains(args.remove, args.from_file, args.out)