#!/usr/bin/env bash
# Downloads every public source into data/raw/. Re-run to refresh.
# Each source is also listed on the methodology page, so anyone can rebuild the numbers.
# URLs checked 22 Nov 2026. ONS file paths change with each edition: update them here.
set -euo pipefail
cd "$(dirname "$0")/../data/raw"

ONS="https://www.ons.gov.uk/file?uri="
get() { echo "fetch $2"; curl -sSfL --retry 3 -A 'Mozilla/5.0' -o "$2" "$1"; }

# DfE Unit for Future Skills, The impact of AI on UK jobs and training (Nov 2023), Annex 1 (AIOE by SOC 2010)
get "https://assets.publishing.service.gov.uk/media/65575c92046ed400148b9ad1/Annex_1_-_Impact_of_AI_on_UK_jobs_and_training_data_sources.xlsx" dfe_annex1.xlsx

# ONS relationship between SOC 2010 and SOC 2020 (July 2021)
get "${ONS}/methodology/classificationsandstandards/standardoccupationalclassificationsoc/soc2020/therelationshipbetweenstandardoccupationalclassification2010andstandardoccupationalclassification2020/soc2010soc2020relationshiptablesjuly2021.xlsx" soc2010_to_2020_jul2021.xlsx

# ONS SOC 2020 coding index, version 14 (Aug 2026): everyday job titles -> SOC 2020 and ISCO-08
get "${ONS}/methodology/classificationsandstandards/standardoccupationalclassificationsoc/soc2020/soc2020volume2codingrulesandconventions/soc2020volume2thecodingindexexcel20260827v4.xlsx" soc2020_coding_index.xlsx

# ONS ASHE Table 14, 2026 provisional (SOC 2020 four-digit)
get "${ONS}/employmentandlabourmarket/peopleinwork/earningsandworkinghours/datasets/occupation4digitsoc2010ashetable14/2026provisional/ashetable142026provisional.zip" ashe14_2026p.zip
rm -rf ashe14 && unzip -q -o ashe14_2026p.zip -d ashe14

# ONS Annual Population Survey via Nomis, NM_218_1: SOC 2020 four-digit, UK and 12 regions, latest 12 months
NOMIS="https://www.nomisweb.co.uk/api/v01/dataset/NM_218_1.data.csv?geography=2092957697,TYPE480&date=latest&jtype=0&ftpt=0&c_sex=0&measure=1&measures=20100,20701&select=date_name,geography_name,geography_code,soc2020_full_code,soc2020_full_name,measures_name,obs_value,obs_status"
get "${NOMIS}&etype=0" aps_nm218.csv             # everyone in employment
get "${NOMIS}&etype=1" aps_nm218_employees.csv   # employees only, for the wage bill (ASHE pay covers employees)

# ONS labour demand volumes by SOC 2020 (Textkernel online job adverts), Jan 2017 to Sep 2026
get "${ONS}/employmentandlabourmarket/peopleinwork/employmentandemployeetypes/datasets/labourdemandvolumesbystandardoccupationclassificationsoc2020uk/january2017toseptember2026/labourdemandbyoccupation.xlsx" ldv_sep2026.xlsx

# ONS Census 2021, occupations by age and sex, England and Wales (four-digit)
# ONS APS ad hoc 3501, four-digit occupation by highest qualification, Apr 2025 to Mar 2026
# ILO Working Paper 140 annex (HTML table only), saved as ilo_wp140_genai_exposure_isco08_4digit.csv
# These three are linked from the methodology page; the copies in data/raw are the files used.

echo done
