Merge branch 'main' into process_report

Author: oree-xx

.github/workflows/1-fetch.yml

@@ -3,13 +3,10 @@ name: Fetch Data
 on:
   schedule:
     # Normal schedule
-    # # at 01:15 on all days in first month of each quarter
-    # - cron: '15 1 * 1,4,7,10 *'
-    # # at 01:15 on days 1-14 in second month of each quarter
-    # - cron: '15 1 1-14 2,5,8,11 *'
-    # Temp schedule
-    # at 01:15 on all days in all months
-    - cron: '15 1 * * *'
+    # # at 03:15 on all days in first month of each quarter
+    - cron: '15 3 * 1,4,7,10 *'
+    # # at 03:15 on days 1-14 in second month of each quarter
+    - cron: '15 3 1-14 2,5,8,11 *'
 
   workflow_dispatch:
 
@@ -29,7 +26,7 @@ jobs:
           git config --global user.email "${{ secrets.BOT_EMAIL }}"
 
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           # Default fetch-depth is 1, however that value results in errors
           # when GitPython attempts to push changes:
@@ -38,7 +35,7 @@ jobs:
           token: ${{ secrets.BOT_TOKEN }}
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: '3.11'
 
@@ -50,20 +47,45 @@ jobs:
         run: |
           pipenv sync --system
 
-      # CC Technology team members:
-      #   See cc-quantifying-bot Google Workspace  entry in Bitwarden for
-      #   information on GCS_ secrets
-      - name: Fetch from Google Custom Search (GCS)
-        run: |
-          ./scripts/1-fetch/gcs_fetch.py \
-            --limit=100 --enable-save --enable-git
-        env:
-          GCS_DEVELOPER_KEY: ${{ secrets.GCS_DEVELOPER_KEY }}
-          GCS_CX: ${{ secrets.GCS_CX }}
+      # Fetch from arXiv disabled due to long run time (~6 hours)
+      #
+      # For now, data is fetched manually :/
+
+      # Fetch from Europeana disabled due to being considered incomplete
+      # https://github.com/creativecommons/quantifying/issues/224
+
+      # Fetch from GCS disabled due to Google blocking GitHub Action runners
+      # # CC Technology team members:
+      # #   See cc-quantifying-bot Google Workspace entry in Bitwarden for
+      # #   information on GCS_ secrets
+      # - name: Fetch from Google Custom Search (GCS)
+      #  run: |
+      #     ./scripts/1-fetch/gcs_fetch.py \
+      #       --limit=100 --enable-save --enable-git
+      #   env:
+      #     GCS_DEVELOPER_KEY: ${{ secrets.GCS_DEVELOPER_KEY }}
+      #     GCS_CX: ${{ secrets.GCS_CX }}
+      #
+      # For now, data is fetched manually :/
 
       - name: Fetch from GitHub
         run: |
           ./scripts/1-fetch/github_fetch.py \
             --enable-save --enable-git
         env:
           GH_TOKEN: ${{ secrets.BOT_TOKEN }}
+
+      # Fetch from Openverse disabled due to limitations of anonymous API
+      # access
+
+      - name: Fetch from Smithsonian
+        run: |
+          ./scripts/1-fetch/smithsonian_fetch.py \
+            --enable-save --enable-git
+        env:
+          DATA_GOV_API_KEY: ${{ secrets.DATA_GOV_API_KEY }}
+
+      - name: Fetch from Wikipedia
+        run: |
+          ./scripts/1-fetch/wikipedia_fetch.py \
+            --enable-save --enable-git

data/2026Q1/1-fetch/arxiv_1_count.csv

@@ -0,0 +1,8 @@
+"TOOL_IDENTIFIER","COUNT"
+"CC BY 3.0","7913"
+"CC BY 4.0","485432"
+"CC BY-NC-ND 4.0","76372"
+"CC BY-NC-SA 3.0","5874"
+"CC BY-NC-SA 4.0","57750"
+"CC BY-SA 4.0","26477"
+"CC0 1.0","19617"

data/2026Q1/1-fetch/arxiv_2_count_by_category_report.csv

@@ -0,0 +1,92 @@
+"TOOL_IDENTIFIER","YEAR","COUNT"
+"CC BY 3.0","2008","3"
+"CC BY 3.0","2009","878"
+"CC BY 3.0","2010","574"
+"CC BY 3.0","2011","640"
+"CC BY 3.0","2012","899"
+"CC BY 3.0","2013","1173"
+"CC BY 3.0","2014","1376"
+"CC BY 3.0","2015","1817"
+"CC BY 3.0","2016","271"
+"CC BY 3.0","2017","73"
+"CC BY 3.0","2018","43"
+"CC BY 3.0","2019","114"
+"CC BY 3.0","2020","15"
+"CC BY 3.0","2021","7"
+"CC BY 3.0","2022","7"
+"CC BY 3.0","2023","14"
+"CC BY 3.0","2024","6"
+"CC BY 3.0","2025","3"
+"CC BY 4.0","2015","572"
+"CC BY 4.0","2016","1507"
+"CC BY 4.0","2017","2083"
+"CC BY 4.0","2018","3897"
+"CC BY 4.0","2019","5051"
+"CC BY 4.0","2020","10676"
+"CC BY 4.0","2021","48129"
+"CC BY 4.0","2022","63751"
+"CC BY 4.0","2023","78739"
+"CC BY 4.0","2024","99166"
+"CC BY 4.0","2025","150068"
+"CC BY 4.0","2026","21793"
+"CC BY-NC-ND 4.0","2020","770"
+"CC BY-NC-ND 4.0","2021","9070"
+"CC BY-NC-ND 4.0","2022","10792"
+"CC BY-NC-ND 4.0","2023","12851"
+"CC BY-NC-ND 4.0","2024","16855"
+"CC BY-NC-ND 4.0","2025","23070"
+"CC BY-NC-ND 4.0","2026","2964"
+"CC BY-NC-SA 3.0","2008","30"
+"CC BY-NC-SA 3.0","2009","631"
+"CC BY-NC-SA 3.0","2010","693"
+"CC BY-NC-SA 3.0","2011","552"
+"CC BY-NC-SA 3.0","2012","659"
+"CC BY-NC-SA 3.0","2013","847"
+"CC BY-NC-SA 3.0","2014","991"
+"CC BY-NC-SA 3.0","2015","1162"
+"CC BY-NC-SA 3.0","2016","114"
+"CC BY-NC-SA 3.0","2017","38"
+"CC BY-NC-SA 3.0","2018","27"
+"CC BY-NC-SA 3.0","2019","73"
+"CC BY-NC-SA 3.0","2020","13"
+"CC BY-NC-SA 3.0","2021","16"
+"CC BY-NC-SA 3.0","2022","10"
+"CC BY-NC-SA 3.0","2023","14"
+"CC BY-NC-SA 3.0","2024","3"
+"CC BY-NC-SA 3.0","2025","1"
+"CC BY-NC-SA 4.0","2015","348"
+"CC BY-NC-SA 4.0","2016","837"
+"CC BY-NC-SA 4.0","2017","809"
+"CC BY-NC-SA 4.0","2018","1585"
+"CC BY-NC-SA 4.0","2019","2092"
+"CC BY-NC-SA 4.0","2020","3115"
+"CC BY-NC-SA 4.0","2021","5560"
+"CC BY-NC-SA 4.0","2022","6557"
+"CC BY-NC-SA 4.0","2023","8230"
+"CC BY-NC-SA 4.0","2024","10997"
+"CC BY-NC-SA 4.0","2025","15624"
+"CC BY-NC-SA 4.0","2026","1996"
+"CC BY-SA 4.0","2015","102"
+"CC BY-SA 4.0","2016","300"
+"CC BY-SA 4.0","2017","304"
+"CC BY-SA 4.0","2018","540"
+"CC BY-SA 4.0","2019","699"
+"CC BY-SA 4.0","2020","1031"
+"CC BY-SA 4.0","2021","2573"
+"CC BY-SA 4.0","2022","3145"
+"CC BY-SA 4.0","2023","3917"
+"CC BY-SA 4.0","2024","5306"
+"CC BY-SA 4.0","2025","7512"
+"CC BY-SA 4.0","2026","1048"
+"CC0 1.0","2015","143"
+"CC0 1.0","2016","374"
+"CC0 1.0","2017","378"
+"CC0 1.0","2018","630"
+"CC0 1.0","2019","761"
+"CC0 1.0","2020","1127"
+"CC0 1.0","2021","2411"
+"CC0 1.0","2022","2725"
+"CC0 1.0","2023","3017"
+"CC0 1.0","2024","3371"
+"CC0 1.0","2025","4155"
+"CC0 1.0","2026","525"

data/2026Q1/1-fetch/arxiv_3_count_by_year.csv

@@ -0,0 +1,36 @@
+"TOOL_IDENTIFIER","AUTHOR_BUCKET","COUNT"
+"CC BY 3.0","1","2045"
+"CC BY 3.0","2","2024"
+"CC BY 3.0","3","1620"
+"CC BY 3.0","4","842"
+"CC BY 3.0","5+","1382"
+"CC BY 4.0","1","63398"
+"CC BY 4.0","2","93597"
+"CC BY 4.0","3","98278"
+"CC BY 4.0","4","73517"
+"CC BY 4.0","5+","156642"
+"CC BY-NC-ND 4.0","1","8480"
+"CC BY-NC-ND 4.0","2","14156"
+"CC BY-NC-ND 4.0","3","15155"
+"CC BY-NC-ND 4.0","4","11935"
+"CC BY-NC-ND 4.0","5+","26646"
+"CC BY-NC-SA 3.0","1","1560"
+"CC BY-NC-SA 3.0","2","1586"
+"CC BY-NC-SA 3.0","3","1225"
+"CC BY-NC-SA 3.0","4","680"
+"CC BY-NC-SA 3.0","5+","823"
+"CC BY-NC-SA 4.0","1","6549"
+"CC BY-NC-SA 4.0","2","9892"
+"CC BY-NC-SA 4.0","3","11253"
+"CC BY-NC-SA 4.0","4","9143"
+"CC BY-NC-SA 4.0","5+","20913"
+"CC BY-SA 4.0","1","3748"
+"CC BY-SA 4.0","2","5108"
+"CC BY-SA 4.0","3","5396"
+"CC BY-SA 4.0","4","4043"
+"CC BY-SA 4.0","5+","8182"
+"CC0 1.0","1","3525"
+"CC0 1.0","2","4421"
+"CC0 1.0","3","4121"
+"CC0 1.0","4","2681"
+"CC0 1.0","5+","4869"

data/2026Q1/1-fetch/arxiv_4_count_by_author_bucket.csv

@@ -0,0 +1,7 @@
+api_description: Open Archives Initiative Protocol for Metadata Havesting (OAI-PMH)
+api_endpoint: https://oaipmh.arxiv.org/oai
+cc_articles_found: 679435
+fetch_limit: -1
+from_add_date: '2008-02-05'
+quarter: 2026Q1
+script: arxiv_fetch.py