From b8581aa7ebeef828708408c8ffdb1b1af9819648 Mon Sep 17 00:00:00 2001 From: Julia Lober Date: Thu, 5 Mar 2026 11:21:03 -0700 Subject: [PATCH 1/7] download slcs --- notebooks/slc_download_test.ipynb | 140 ++++++++++++++ uavsar_pytools/download/download.py | 6 +- uavsar_pytools/download/download_slcs.py | 225 +++++++++++++++++++++++ 3 files changed, 369 insertions(+), 2 deletions(-) create mode 100644 notebooks/slc_download_test.ipynb create mode 100644 uavsar_pytools/download/download_slcs.py diff --git a/notebooks/slc_download_test.ipynb b/notebooks/slc_download_test.ipynb new file mode 100644 index 0000000..f7191cc --- /dev/null +++ b/notebooks/slc_download_test.ipynb @@ -0,0 +1,140 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "81d1d1db", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from uavsar_pytools.download.download_slcs import get_uavsar_slcs, download_uavsar_slcs" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "cedb8cea", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created test directory at /Users/JULIALOBER/uavsar_pytools/notebooks/uavsar_test_downloads\n" + ] + } + ], + "source": [ + "# Create a temporary directory for our test downloads\n", + "test_dir = './uavsar_test_downloads'\n", + "os.makedirs(test_dir, exist_ok=True)\n", + "test_dir = os.path.abspath(test_dir)\n", + "print(f\"Created test directory at {test_dir}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "734e76cb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generated 16 URLs for grmesa_09305.\n", + "\n", + "First 3 URLs found:\n", + "grmesa_09305_20013_005_200226_L090HH_01_BU_s1_2x8.slc\n", + "grmesa_09305_20013_005_200226_L090HH_01_BU.ann\n", + "grmesa_09305_20013_005_200226_L090HH_01_BU_s2_2x8.slc\n" + ] + } + ], + "source": [ + "# Query links for a narrow time window\n", + "test_links_dict = get_uavsar_slcs(\n", + " flight_name='grmesa',\n", + " start_date='2020-02-01',\n", + " end_date='2020-02-28',\n", + " getann=True \n", + ")\n", + "\n", + "# Extract the dictionary key (should be 'grmesa_03913' or similar)\n", + "test_key = list(test_links_dict.keys())[0]\n", + "generated_urls = test_links_dict[test_key]\n", + "\n", + "print(f\"Generated {len(generated_urls)} URLs for {test_key}.\")\n", + "print(\"\\nFirst 3 URLs found:\")\n", + "for url in generated_urls[:3]:\n", + " print(url)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "046d76ae", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Original URL: grmesa_09305_20013_005_200226_L090HH_01_BU.ann\n", + "Testing download pipeline with: grmesa_09305_20013_005_200226_L090HH_01_BU.ann\n", + "Found file verified on disk at /Users/JULIALOBER/uavsar_pytools/notebooks/uavsar_test_downloads/grmesa_09305_20013_005_200226_L090HH_01_BU.ann\n" + ] + } + ], + "source": [ + "# Filter the list to ONLY include an .ann file to save bandwidth\n", + "ann_files_only = [url for url in generated_urls if url.endswith('.ann')]\n", + "\n", + "if not ann_files_only:\n", + " print(\"No .ann files found. Make sure getann=True was passed in Cell 3.\")\n", + "else:\n", + " # We will just pass the very first .ann file to the download function\n", + " test_download = [ann_files_only[0]]\n", + " \n", + " # Strip the base URL path to match what download_uavsar_jpl expects\n", + " # The function expects just the filename, e.g. \"grmesa_03913_..._BU.ann\"\n", + " # filename_only = [test_download_list[0].split('/')[-1]]\n", + " print(f\"Original URL: {test_download[0]}\")\n", + " \n", + " print(f\"Testing download pipeline with: {test_download[0]}\")\n", + " \n", + " # Run the download\n", + " download_uavsar_slcs(test_download, out_dir=test_dir)\n", + " \n", + " # Verify it exists\n", + " downloaded_path = os.path.join(test_dir, test_download[0])\n", + " if os.path.exists(downloaded_path):\n", + " print(f\"Found file verified on disk at {downloaded_path}\")\n", + " else:\n", + " print(\"Download failed or file not found.\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "uavsar_pytools", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/uavsar_pytools/download/download.py b/uavsar_pytools/download/download.py index 8780d16..3988358 100644 --- a/uavsar_pytools/download/download.py +++ b/uavsar_pytools/download/download.py @@ -11,7 +11,9 @@ from os.path import join, isdir, isfile, basename, dirname, exists from tqdm.auto import tqdm import logging - +import urllib +from collections import defaultdict +import asf_search as asf import time log = logging.getLogger(__name__) @@ -155,4 +157,4 @@ def download_zip(url, output_dir): else: log.info(f'{local} already exists, skipping download!') - return local + return local \ No newline at end of file diff --git a/uavsar_pytools/download/download_slcs.py b/uavsar_pytools/download/download_slcs.py new file mode 100644 index 0000000..724e3ac --- /dev/null +++ b/uavsar_pytools/download/download_slcs.py @@ -0,0 +1,225 @@ +import os +import logging +import numpy as np +import asf_search as asf +import requests +from collections import defaultdict + +# Import the native download function from the repo +from uavsar_pytools.download.download import stream_download + +# Initialize the logger +log = logging.getLogger(__name__) + +def get_uavsar_slcs( + flight_name: str, + flight_num: str = None, + getann: bool = False, + start_date: str = '2020-01-01', + end_date: str = '2021-12-31', + pol: list = ['HH'], + sec: list = ['s1', 's2', 's3'], + pxspc: str = '2x8', + tag: list = ['BU'] +) -> dict: + """ + Query the ASF DAAC for UAVSAR flight lines and generate a dictionary of JPL download URLs. + + This function maps a given UAVSAR campaign name to its corresponding ASF search string, + retrieves the metadata for the specified date range, and constructs the expected filenames + for the JPL Release 30 data portal. + + Parameters + ---------- + flight_name : str + The abbreviation or full name of the UAVSAR campaign (e.g., 'lowman' or 'Lowman, CO'). + flight_num : str, optional + Specific flight line number to filter the search. Default is None (returns all lines). + getann : bool, optional + If True, appends the annotation (.ann) file URL for each flight. Default is False. + start_date : str, optional + Start date for the ASF search in 'YYYY-MM-DD' format. Default is '2020-01-01'. + end_date : str, optional + End date for the ASF search in 'YYYY-MM-DD' format. Default is '2021-12-31'. + pol : list of str, optional + List of polarization bands to include. Default is ['HH']. + sec : list of str, optional + List of data segments/swaths to include. Default is ['s1', 's2', 's3']. + pxspc : str, optional + Pixel spacing string to append to the filename. Default is '2x8'. + tag : list of str, optional + List of file type tags to include (e.g., 'BU' for baseline-updated). Default is ['BU']. + + Returns + ------- + dict + A defaultdict where keys are formatted as '{flight_abbr}_{flight_line}' and values + are lists of constructed JPL filenames/URLs. + + Raises + ------ + ValueError + If the provided `flight_name` is not found in the valid campaigns mapping. + """ + jpl_site = 'https://downloaduav2.jpl.nasa.gov' + release_folder = 'Release30' + links = defaultdict(list) + + campaigns = { + 'grmesa': 'Grand Mesa, CO', + 'lowman': 'Lowman, CO', + 'fraser': 'Fraser, CO', + 'ironto': 'Ironton, CO', # Senator Beck Basin + 'peeler': 'Peeler Peak, CO', # East River + 'rockmt': 'Rocky Mountains NP, CO', # Cameron Pass + 'silver': 'Silver City, ID', # Reynolds Creek + 'uticam': 'Utica, MT', # Central Ag Research Center + 'saltla': 'Salt Lake City, UT', # Little Cottonwood Canyon + 'losala': 'Los Alamos, NM', # Jemez River + 'dorado': 'Eldorado National Forest, CA', # American River Basin + 'donner': 'Donner Memorial State Park, CA', # Sagehen Creek + 'sierra': 'Sierra National Forest, CA' # Lakes Basin + } + + if flight_name not in campaigns.values(): + try: + flight_abbr = flight_name + flight_name = campaigns[flight_name] + except KeyError: + raise ValueError(f"Invalid flight name: {flight_name}. Valid options are: {campaigns}") + else: + flight_abbr = next((k for k, v in campaigns.items() if v == flight_name), None) + + log.info(f'Getting files for {flight_name} ({flight_abbr}).') + + if flight_num is None: + grans = asf.search(platform='UAVSAR', + campaign=flight_name, + beamMode='POL', + start=start_date, + end=end_date) + else: + grans = asf.search(platform='UAVSAR', + campaign=flight_name, + flightLine=flight_num, + beamMode='POL', + start=start_date, + end=end_date) + + log.info(f"{len(grans)} granules found for {flight_name}") + + flight_lines = set() + + for g in grans: + scene_name = g.properties["sceneName"] + parts = scene_name.split("_") + + site = parts[1] + flight_line = parts[2].zfill(5) + flight_lines.add(flight_line) + + flight1_id = parts[3] + '_' + parts[4] + band = parts[6] + version = parts[8] + date1 = parts[5] + + for t in tag: + for p in pol: + for s in sec: + f1_base = f"{site}_{flight_line}_{flight1_id}_{date1}_{band}{p}_{version}_{t}" + + stack_dir = f"{site}_{flight_line}_{version}" + base_url = f"{jpl_site}/{release_folder}/{stack_dir}" + + urls = [ + f"{f1_base}_{s}_{pxspc}.slc", + ] + + if getann: + urls.append(f"{f1_base}.ann") + + dict_key = f'{flight_abbr}_{flight_line}' + for url in urls: + if url not in links[dict_key]: + links[dict_key].append(url) + + return links + + +def download_uavsar_slcs(files: list, out_dir: str): + """ + Download UAVSAR files from the JPL data portal to a specified local directory. + + This function constructs the download URL for each file, checks for proper release folders + (handling 404 HTML redirects from JPL gracefully), and utilizes the package's native + `stream_download` method to fetch the data. It skips files that already exist locally. + + Parameters + ---------- + files : list of str + A list of UAVSAR filenames generated by `get_uavsar_slcs`. + out_dir : str + The absolute or relative path to the local directory where files will be saved. + + Returns + ------- + None + Outputs are saved directly to disk. Function logs the progress and status of downloads. + """ + def is_html(link): + try: + # stream=True fetches the headers without downloading the whole file + response = requests.get(link, stream=True) + content_type = response.headers.get('Content-Type', '') + response.close() + return 'text/html' in content_type + except requests.RequestException as e: + log.error(f"Failed to check link {link}: {e}") + return False + + BASE_URL = 'https://downloaduav2.jpl.nasa.gov' + releases = np.arange(26, 32)[::-1] + RELEASE_FOLDERS = [f'Release{r}' for r in releases] + + # Guard clause in case an empty list is passed + if not files: + log.warning("No files provided to download.") + return + + try: + parts = files[0].split('_') + flight_folder = f"{parts[0]}_{parts[1]}_{parts[6]}" + except IndexError as e: + log.error(f"Filename {files[0]} was not recognized as a valid UAVSAR filename.") + return + + # Find valid release folder + release_folder = None + for r in RELEASE_FOLDERS: + url = f'{BASE_URL}/{r}/{flight_folder}/{files[0]}' + if not is_html(url): + log.info(f'Found valid link, using release folder {r} for download.') + release_folder = r + break + + if not release_folder: + log.error("Could not find a valid release folder for these files.") + return + + # Download routine + for f in files: + link = f'{BASE_URL}/{release_folder}/{flight_folder}/{f}' + + if is_html(link): + log.warning(f"Link {link} appears to be an HTML page. Skipping download.") + continue + + filename = os.path.join(out_dir, f) + log.info(f'Checking for {filename}') + + if os.path.exists(filename): + log.info(f"File {filename} already exists. Skipping download.") + continue + + log.info(f"Downloading {f} to {out_dir}...") + stream_download(link, f"{out_dir}/{f}") \ No newline at end of file From 3e38edf4eb0d7fcd3da90edefafa761549c4585d Mon Sep 17 00:00:00 2001 From: Julia Lober Date: Thu, 5 Mar 2026 12:29:09 -0700 Subject: [PATCH 2/7] added dop/lkv/llh filenames --- notebooks/slc_download_test.ipynb | 84 ++++++++++++++++++------ uavsar_pytools/download/download_slcs.py | 76 +++++++++++++-------- 2 files changed, 110 insertions(+), 50 deletions(-) diff --git a/notebooks/slc_download_test.ipynb b/notebooks/slc_download_test.ipynb index f7191cc..093beab 100644 --- a/notebooks/slc_download_test.ipynb +++ b/notebooks/slc_download_test.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "81d1d1db", "metadata": {}, "outputs": [], @@ -13,7 +13,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 2, "id": "cedb8cea", "metadata": {}, "outputs": [ @@ -35,7 +35,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "734e76cb", "metadata": {}, "outputs": [ @@ -43,22 +43,35 @@ "name": "stdout", "output_type": "stream", "text": [ - "Generated 16 URLs for grmesa_09305.\n", + "Generated 11 URLs for lowman_23205.\n", "\n", "First 3 URLs found:\n", - "grmesa_09305_20013_005_200226_L090HH_01_BU_s1_2x8.slc\n", - "grmesa_09305_20013_005_200226_L090HH_01_BU.ann\n", - "grmesa_09305_20013_005_200226_L090HH_01_BU_s2_2x8.slc\n" + "lowman_23205_20011_003_200221_L090HH_01_BC_s1_2x8.slc\n", + "lowman_23205_01_BC_s1_2x8.llh\n", + "lowman_23205_01_BC_s1_2x8.lkv\n", + "lowman_23205_20011_003_200221_L090HH_01_BC_s2_2x8.slc\n", + "lowman_23205_01_BC_s2_2x8.llh\n", + "lowman_23205_01_BC_s2_2x8.lkv\n", + "lowman_23205_20011_003_200221_L090HH_01_BC.ann\n", + "lowman_23205_01_BC.dop\n", + "lowman_23205_20007_003_200213_L090HH_01_BC_s1_2x8.slc\n", + "lowman_23205_20007_003_200213_L090HH_01_BC_s2_2x8.slc\n", + "lowman_23205_20007_003_200213_L090HH_01_BC.ann\n" ] } ], "source": [ "# Query links for a narrow time window\n", "test_links_dict = get_uavsar_slcs(\n", - " flight_name='grmesa',\n", + " flight_name='lowman',\n", " start_date='2020-02-01',\n", " end_date='2020-02-28',\n", - " getann=True \n", + " tag=['BC'],\n", + " seg=['s1', 's2'],\n", + " getann=True,\n", + " getdop=True,\n", + " getllh=True,\n", + " getlkv=True\n", ")\n", "\n", "# Extract the dictionary key (should be 'grmesa_03913' or similar)\n", @@ -67,13 +80,13 @@ "\n", "print(f\"Generated {len(generated_urls)} URLs for {test_key}.\")\n", "print(\"\\nFirst 3 URLs found:\")\n", - "for url in generated_urls[:3]:\n", + "for url in generated_urls[:]:\n", " print(url)" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "046d76ae", "metadata": {}, "outputs": [ @@ -81,9 +94,43 @@ "name": "stdout", "output_type": "stream", "text": [ - "Original URL: grmesa_09305_20013_005_200226_L090HH_01_BU.ann\n", - "Testing download pipeline with: grmesa_09305_20013_005_200226_L090HH_01_BU.ann\n", - "Found file verified on disk at /Users/JULIALOBER/uavsar_pytools/notebooks/uavsar_test_downloads/grmesa_09305_20013_005_200226_L090HH_01_BU.ann\n" + "Original URL: lowman_23205_20011_003_200221_L090HH_01_BC.ann\n", + "Testing download pipeline with: lowman_23205_20011_003_200221_L090HH_01_BC.ann\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a3224d50330c427a93af812fe9e5e8d8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Downloading lowman_23205_20011_003_200221_L090HH_01_BC.ann: 0%| | 0.00/47.0k [00:00 dict: """ @@ -45,8 +48,8 @@ def get_uavsar_slcs( List of polarization bands to include. Default is ['HH']. sec : list of str, optional List of data segments/swaths to include. Default is ['s1', 's2', 's3']. - pxspc : str, optional - Pixel spacing string to append to the filename. Default is '2x8'. + pxlsp : list of str, optional + Pixel spacing strings to append to the filename. Default is ['2x8']. tag : list of str, optional List of file type tags to include (e.g., 'BU' for baseline-updated). Default is ['BU']. @@ -61,11 +64,11 @@ def get_uavsar_slcs( ValueError If the provided `flight_name` is not found in the valid campaigns mapping. """ - jpl_site = 'https://downloaduav2.jpl.nasa.gov' - release_folder = 'Release30' + # jpl_site = 'https://downloaduav2.jpl.nasa.gov' + # release_folder = 'Release30' links = defaultdict(list) - campaigns = { + campaigns = { # SnowEx campaigns and abbreviations 'grmesa': 'Grand Mesa, CO', 'lowman': 'Lowman, CO', 'fraser': 'Fraser, CO', @@ -125,23 +128,31 @@ def get_uavsar_slcs( for t in tag: for p in pol: - for s in sec: - f1_base = f"{site}_{flight_line}_{flight1_id}_{date1}_{band}{p}_{version}_{t}" - - stack_dir = f"{site}_{flight_line}_{version}" - base_url = f"{jpl_site}/{release_folder}/{stack_dir}" - - urls = [ - f"{f1_base}_{s}_{pxspc}.slc", - ] - - if getann: - urls.append(f"{f1_base}.ann") - - dict_key = f'{flight_abbr}_{flight_line}' - for url in urls: - if url not in links[dict_key]: - links[dict_key].append(url) + urls = [] + for s in seg: + for pxl in pxlsp: + f1_base = f"{site}_{flight_line}_{flight1_id}_{date1}_{band}{p}_{version}_{t}" + + # stack_dir = f"{site}_{flight_line}_{version}" + # base_url = f"{jpl_site}/{release_folder}/{stack_dir}" + + urls.append(f"{f1_base}_{s}_{pxl}.slc") + + # this will cause some repeats, since there is only one per seg + if getllh: + urls.append(f"{site}_{flight_line}_{version}_{t}_{s}_{pxl}.llh") + if getlkv: + urls.append(f"{site}_{flight_line}_{version}_{t}_{s}_{pxl}.lkv") + + if getann: + urls.append(f"{f1_base}.ann") + if getdop: + urls.append(f"{site}_{flight_line}_{version}_{t}.dop") + + dict_key = f'{flight_abbr}_{flight_line}' + for url in urls: + if url not in links[dict_key]: + links[dict_key].append(url) return links @@ -168,7 +179,7 @@ def download_uavsar_slcs(files: list, out_dir: str): """ def is_html(link): try: - # stream=True fetches the headers without downloading the whole file + # stream=true fetches the headers without downloading the whole file response = requests.get(link, stream=True) content_type = response.headers.get('Content-Type', '') response.close() @@ -181,19 +192,26 @@ def is_html(link): releases = np.arange(26, 32)[::-1] RELEASE_FOLDERS = [f'Release{r}' for r in releases] - # Guard clause in case an empty list is passed + # check for empty list if not files: log.warning("No files provided to download.") return + if type(files) is not list: + log.error(f"Files parameter should be a list of filenames, got type {type(files)} instead.") + return + if type(files[0]) is not str: + log.error(f"Files list should contain strings (filenames), got type {type(files[0])} instead.") + return + # very basic check for filename structure try: parts = files[0].split('_') flight_folder = f"{parts[0]}_{parts[1]}_{parts[6]}" - except IndexError as e: + except: log.error(f"Filename {files[0]} was not recognized as a valid UAVSAR filename.") return - # Find valid release folder + # find valid release folder release_folder = None for r in RELEASE_FOLDERS: url = f'{BASE_URL}/{r}/{flight_folder}/{files[0]}' @@ -206,7 +224,7 @@ def is_html(link): log.error("Could not find a valid release folder for these files.") return - # Download routine + # download files for f in files: link = f'{BASE_URL}/{release_folder}/{flight_folder}/{f}' From 46907bc629c8983629bd11a0224c3d4560551d69 Mon Sep 17 00:00:00 2001 From: Julia Lober Date: Sun, 8 Mar 2026 10:21:55 -0600 Subject: [PATCH 3/7] expand Release folder search --- uavsar_pytools/download/download_slcs.py | 30 ++++++++++++++---------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/uavsar_pytools/download/download_slcs.py b/uavsar_pytools/download/download_slcs.py index 99ee307..e41e9e6 100644 --- a/uavsar_pytools/download/download_slcs.py +++ b/uavsar_pytools/download/download_slcs.py @@ -3,8 +3,10 @@ import numpy as np import asf_search as asf import requests +import datetime from collections import defaultdict + # Import the native download function from the repo from uavsar_pytools.download.download import stream_download @@ -23,7 +25,7 @@ def get_uavsar_slcs( pol: list = ['HH'], seg: list = ['s1', 's2', 's3'], pxlsp: list = ['2x8'], - tag: list = ['BU'] + tag: list = ['BU', 'BC'] ) -> dict: """ Query the ASF DAAC for UAVSAR flight lines and generate a dictionary of JPL download URLs. @@ -72,13 +74,13 @@ def get_uavsar_slcs( 'grmesa': 'Grand Mesa, CO', 'lowman': 'Lowman, CO', 'fraser': 'Fraser, CO', - 'ironto': 'Ironton, CO', # Senator Beck Basin + 'irnton': 'Ironton, CO', # Senator Beck Basin 'peeler': 'Peeler Peak, CO', # East River 'rockmt': 'Rocky Mountains NP, CO', # Cameron Pass 'silver': 'Silver City, ID', # Reynolds Creek 'uticam': 'Utica, MT', # Central Ag Research Center - 'saltla': 'Salt Lake City, UT', # Little Cottonwood Canyon - 'losala': 'Los Alamos, NM', # Jemez River + 'stlake': 'Salt Lake City, UT', # Little Cottonwood Canyon + 'alamos': 'Los Alamos, NM', # Jemez River 'dorado': 'Eldorado National Forest, CA', # American River Basin 'donner': 'Donner Memorial State Park, CA', # Sagehen Creek 'sierra': 'Sierra National Forest, CA' # Lakes Basin @@ -92,7 +94,9 @@ def get_uavsar_slcs( raise ValueError(f"Invalid flight name: {flight_name}. Valid options are: {campaigns}") else: flight_abbr = next((k for k, v in campaigns.items() if v == flight_name), None) - + + current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + log.info(f"{current_time}") log.info(f'Getting files for {flight_name} ({flight_abbr}).') if flight_num is None: @@ -189,7 +193,7 @@ def is_html(link): return False BASE_URL = 'https://downloaduav2.jpl.nasa.gov' - releases = np.arange(26, 32)[::-1] + releases = np.arange(20, 40)[::-1] RELEASE_FOLDERS = [f'Release{r}' for r in releases] # check for empty list @@ -214,11 +218,13 @@ def is_html(link): # find valid release folder release_folder = None for r in RELEASE_FOLDERS: - url = f'{BASE_URL}/{r}/{flight_folder}/{files[0]}' - if not is_html(url): - log.info(f'Found valid link, using release folder {r} for download.') - release_folder = r - break + for i in range(5): + url = f'{BASE_URL}/{r}/{flight_folder}/{files[i]}' + log.info(f"Trying release folder {r}: {url}") + if not is_html(url): + log.info(f'Found valid link, using release folder {r} for download.') + release_folder = r + break if not release_folder: log.error("Could not find a valid release folder for these files.") @@ -240,4 +246,4 @@ def is_html(link): continue log.info(f"Downloading {f} to {out_dir}...") - stream_download(link, f"{out_dir}/{f}") \ No newline at end of file + stream_download(link, f"{out_dir}/{f}") From a41d8d7689c14e34330f00e8bed1c763494b17d3 Mon Sep 17 00:00:00 2001 From: Julia Lober Date: Sun, 8 Mar 2026 10:28:03 -0600 Subject: [PATCH 4/7] added tag to seach --- uavsar_pytools/download/download_slcs.py | 75 +++++++++++++----------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/uavsar_pytools/download/download_slcs.py b/uavsar_pytools/download/download_slcs.py index e41e9e6..d82a0f5 100644 --- a/uavsar_pytools/download/download_slcs.py +++ b/uavsar_pytools/download/download_slcs.py @@ -24,8 +24,8 @@ def get_uavsar_slcs( end_date: str = '2021-12-31', pol: list = ['HH'], seg: list = ['s1', 's2', 's3'], - pxlsp: list = ['2x8'], - tag: list = ['BU', 'BC'] + pxlsp: list = ['2x8'] + # tag: list = ['BU', 'BC'] ) -> dict: """ Query the ASF DAAC for UAVSAR flight lines and generate a dictionary of JPL download URLs. @@ -130,33 +130,33 @@ def get_uavsar_slcs( version = parts[8] date1 = parts[5] - for t in tag: - for p in pol: - urls = [] - for s in seg: - for pxl in pxlsp: - f1_base = f"{site}_{flight_line}_{flight1_id}_{date1}_{band}{p}_{version}_{t}" - - # stack_dir = f"{site}_{flight_line}_{version}" - # base_url = f"{jpl_site}/{release_folder}/{stack_dir}" - - urls.append(f"{f1_base}_{s}_{pxl}.slc") - - # this will cause some repeats, since there is only one per seg - if getllh: - urls.append(f"{site}_{flight_line}_{version}_{t}_{s}_{pxl}.llh") - if getlkv: - urls.append(f"{site}_{flight_line}_{version}_{t}_{s}_{pxl}.lkv") - - if getann: - urls.append(f"{f1_base}.ann") - if getdop: - urls.append(f"{site}_{flight_line}_{version}_{t}.dop") - - dict_key = f'{flight_abbr}_{flight_line}' - for url in urls: - if url not in links[dict_key]: - links[dict_key].append(url) + # for t in tag: + for p in pol: + urls = [] + for s in seg: + for pxl in pxlsp: + f1_base = f"{site}_{flight_line}_{flight1_id}_{date1}_{band}{p}_{version}_[BC/BU]" + + # stack_dir = f"{site}_{flight_line}_{version}" + # base_url = f"{jpl_site}/{release_folder}/{stack_dir}" + + urls.append(f"{f1_base}_{s}_{pxl}.slc") + + # this will cause some repeats, since there is only one per seg + if getllh: + urls.append(f"{site}_{flight_line}_{version}_[BC/BU]_{s}_{pxl}.llh") + if getlkv: + urls.append(f"{site}_{flight_line}_{version}_[BC/BU]_{s}_{pxl}.lkv") + + if getann: + urls.append(f"{f1_base}.ann") + if getdop: + urls.append(f"{site}_{flight_line}_{version}_[BC/BU].dop") + + dict_key = f'{flight_abbr}_{flight_line}' + for url in urls: + if url not in links[dict_key]: + links[dict_key].append(url) return links @@ -217,21 +217,30 @@ def is_html(link): # find valid release folder release_folder = None + tag = None for r in RELEASE_FOLDERS: - for i in range(5): - url = f'{BASE_URL}/{r}/{flight_folder}/{files[i]}' - log.info(f"Trying release folder {r}: {url}") + # for i in range(5): + url = f'{BASE_URL}/{r}/{flight_folder}/{files[i]}' + tags = ['BU', 'BC'] + for t in tags: + url = url.replace('[BC/BU]', t) + log.info(f"Trying release folder {r} and tag {t}: {url}") if not is_html(url): - log.info(f'Found valid link, using release folder {r} for download.') + log.info(f'Found valid link, using {r}, {t} for download.') release_folder = r + tag = t break if not release_folder: log.error("Could not find a valid release folder for these files.") return + elif not tag: + log.error("Could not determine the correct tag (BU/BC) for these files.") + return # download files for f in files: + f = f.replace('[BC/BU]', tag) link = f'{BASE_URL}/{release_folder}/{flight_folder}/{f}' if is_html(link): From 46a45611a4002c2a254d3557281db86d47976ca7 Mon Sep 17 00:00:00 2001 From: Julia Lober Date: Sun, 8 Mar 2026 10:55:10 -0600 Subject: [PATCH 5/7] finalize BC/BU adjustment --- uavsar_pytools/download/download_slcs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/uavsar_pytools/download/download_slcs.py b/uavsar_pytools/download/download_slcs.py index d82a0f5..c8630cd 100644 --- a/uavsar_pytools/download/download_slcs.py +++ b/uavsar_pytools/download/download_slcs.py @@ -25,7 +25,6 @@ def get_uavsar_slcs( pol: list = ['HH'], seg: list = ['s1', 's2', 's3'], pxlsp: list = ['2x8'] - # tag: list = ['BU', 'BC'] ) -> dict: """ Query the ASF DAAC for UAVSAR flight lines and generate a dictionary of JPL download URLs. @@ -220,9 +219,9 @@ def is_html(link): tag = None for r in RELEASE_FOLDERS: # for i in range(5): - url = f'{BASE_URL}/{r}/{flight_folder}/{files[i]}' tags = ['BU', 'BC'] - for t in tags: + for t in tags: + url = f'{BASE_URL}/{r}/{flight_folder}/{files[0]}' url = url.replace('[BC/BU]', t) log.info(f"Trying release folder {r} and tag {t}: {url}") if not is_html(url): From ee1295a545a5b76befded6d9907ae1ec6d60fe84 Mon Sep 17 00:00:00 2001 From: Julia Lober Date: Fri, 27 Mar 2026 12:43:39 -0600 Subject: [PATCH 6/7] add sleep and ignore_errors to cleanup step for issues on HPC --- uavsar_pytools/georeference.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/uavsar_pytools/georeference.py b/uavsar_pytools/georeference.py index f45d15f..ae974fc 100644 --- a/uavsar_pytools/georeference.py +++ b/uavsar_pytools/georeference.py @@ -197,7 +197,9 @@ def geolocate_uavsar(in_fp, ann_fp, out_dir, llh_fp): to the pixels created by the conversion along the edge of topography.\n\ Error message is known and should not be an issue.') - shutil.rmtree(tmp_dir) + import time + time.sleep(5) + shutil.rmtree(tmp_dir, ignore_errors=True) return res_f From a71df925c7685d97bc6ebb889999b924e8cc8df9 Mon Sep 17 00:00:00 2001 From: Julia Lober Date: Tue, 30 Jun 2026 10:00:07 -0600 Subject: [PATCH 7/7] auto-detect slc spacing and linearly interpolate from 2x8 llh file if needed --- uavsar_pytools/georeference.py | 67 ++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/uavsar_pytools/georeference.py b/uavsar_pytools/georeference.py index ae974fc..1ce1f8c 100644 --- a/uavsar_pytools/georeference.py +++ b/uavsar_pytools/georeference.py @@ -110,10 +110,51 @@ def geolocate_uavsar(in_fp, ann_fp, out_dir, llh_fp): dst.write(arr.astype(arr.dtype), 1) # Add VRT file for each tif - tifs = glob(join(tmp_dir, '*.tif')) # list all .llh files - for tiff in tifs: # loop to open and translate .llh to .vrt, and save .vrt using gdal - raster_dataset = gdal.Open(tiff, gdal.GA_ReadOnly) # read in rasters - raster = gdal.Translate(join(tmp_dir, basename(tiff).replace('.tif','.vrt')), raster_dataset, format = 'VRT', outputType = gdal.GDT_Float32) +# tifs = glob(join(tmp_dir, '*.tif')) # list all .llh files +# for tiff in tifs: # loop to open and translate .llh to .vrt, and save .vrt using gdal +# raster_dataset = gdal.Open(tiff, gdal.GA_ReadOnly) # read in rasters +# raster = gdal.Translate(join(tmp_dir, basename(tiff).replace('.tif','.vrt')), raster_dataset, format = 'VRT', outputType = gdal.GDT_Float32) +# Detect target grid resolution dynamically from input filename (e.g., '1x1', '2x8') + # Detect target grid resolution dynamically from the input filename + if '1x1' in basename(in_fp): + target_spacing = '1x1' + elif '1x4' in basename(in_fp): + target_spacing = '1x4' + else: + target_spacing = '2x8' + + # Fetch target dimensions safely depending on file type (read metadata directly for VRTs) + if ext in ['slc', 'lkv']: + t_rows = desc.get(f'{ext}_1_{target_spacing} rows', {}).get('value', nrows) + t_cols = desc.get(f'{ext}_1_{target_spacing} columns', {}).get('value', ncols) + elif ext == 'vrt': + with rio.open(in_fp) as src: + t_rows = src.height + t_cols = src.width + else: + t_rows, t_cols = nrows, ncols + + # Add VRT file for each tif, applying bilinear interpolation if grid sizes don't match + tifs = glob(join(tmp_dir, '*.tif')) + for tiff in tifs: + raster_dataset = gdal.Open(tiff, gdal.GA_ReadOnly) + if t_rows != nrows or t_cols != ncols: + raster = gdal.Translate( + join(tmp_dir, basename(tiff).replace('.tif','.vrt')), + raster_dataset, + format='VRT', + outputType=gdal.GDT_Float32, + width=t_cols, + height=t_rows, + resampleAlg=gdal.GRA_Bilinear + ) + else: + raster = gdal.Translate( + join(tmp_dir, basename(tiff).replace('.tif','.vrt')), + raster_dataset, + format='VRT', + outputType=gdal.GDT_Float32 + ) raster_dataset = None vrts = glob(join(tmp_dir, '*.vrt')) @@ -132,9 +173,7 @@ def geolocate_uavsar(in_fp, ann_fp, out_dir, llh_fp): } if ext == 'slc': - spacing = in_fp.replace(f'.{ext}','')[-3:] - nrows = desc[f'{ext}_1_{spacing} rows']['value'] - ncols = desc[f'{ext}_1_{spacing} columns']['value'] + nrows, ncols = t_rows, t_cols dtype = np.complex64 arr = np.fromfile(in_fp, dtype = dtype).reshape(nrows, ncols) d_arrs = {} @@ -142,9 +181,7 @@ def geolocate_uavsar(in_fp, ann_fp, out_dir, llh_fp): d_arrs['imag'] = arr.imag elif ext == 'lkv': - spacing = in_fp.replace(f'.{ext}','')[-3:] - nrows = desc[f'{ext}_1_{spacing} rows']['value'] - ncols = desc[f'{ext}_1_{spacing} columns']['value'] + nrows, ncols = t_rows, t_cols dtype = np.dtype('