From 990987eeaf2aa30cc17afef5835210649857ece0 Mon Sep 17 00:00:00 2001 From: Synkevych Roman Date: Thu, 25 Aug 2022 15:16:43 +0300 Subject: [PATCH 1/5] downloads correct forecast data --- template/downloads_grib_test.sh | 64 +++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100755 template/downloads_grib_test.sh diff --git a/template/downloads_grib_test.sh b/template/downloads_grib_test.sh new file mode 100755 index 0000000..d7addf8 --- /dev/null +++ b/template/downloads_grib_test.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +if [ $# -ne 2 ]; then echo "Usage: $0 From, To YYYYMMDD YYYYMMDD"; exit 1; fi + +FROM_YY=$(date -d "$1" '+%Y'); FROM_MM=$(date -d "$1" '+%m'); FROM_DD=$(date -d "$1" '+%d') +TO_YEAR=$(date -d "$2" '+%Y'); TO_MONTH=$(date -d "$2" '+%m'); TO_DAY=$(date -d "$2" '+%d') + +FROM_to_sec=$(date --date="$1" +%s); TO_to_sec=$(date --date="$2" +%s) +AMOUNT_OF_DAYS=$(( (TO_to_sec - FROM_to_sec)/86400)) + +HOURS=(0000 0600 1200 1800 0000) +URL="https://www.ncei.noaa.gov/data/global-forecast-system/access/grid-003-1.0-degree/" +PREFIX="gfs_3_" + +# if YYMM <= 7 month from current date use first forecast else analysis +now=$(date); active_date=$(date --date="$now -7 month" +%s); date_from=$(date --date="$1" +%s) + +if [ $date_from -ge $active_date ]; then + DOMAIN="${URL}forecast/" +else + DOMAIN="${URL}analysis/" +fi + +# if folder does not exist create them +[ -d grib_data ] || mkdir grib_data + +for i in `seq 0 $AMOUNT_OF_DAYS`; do + # if day less than 10 add 0 before + YYMMDD="${FROM_YY}${FROM_MM}${FROM_DD}" + + # at first step download 0000_000 forecast + if [ "$i" -eq 0 ]; then + FILE=${PREFIX}${YYMMDD}"_0000_000.grb2" + URL="${DOMAIN}${FROM_YY}${FROM_MM}/${YYMMDD}" + + wget -c "${URL}/${FILE}" -P "grib_data" + echo "${URL}/${FILE}" >> grib_data/file_for_downloads.txt + echo "${YYMMDD} 000000 ${FILE} ON DISC" >> AVAILABLE + fi + # set the date for the next day + date=$(date --date="$1 + $i day"); + FROM_YY=$(date -d "$date" '+%Y'); FROM_MM=$(date -d "$date" '+%m'); FROM_DD=$(date -d "$date" '+%d') + YYMMDD="${FROM_YY}${FROM_MM}${FROM_DD}" + # download 4 forecasts for every day + + for j in `seq 0 3`; do + # if hour 1800 increment day + PR_DD=${FROM_DD} + if [ "$j" -eq 3 ]; then + PR_date=$(date -d "$date + 1 day") + PR_DD=$(date -d "$PR_date" '+%d') + fi + + HH="${HOURS[j]}" + iterator=$((j+1)) + PR_HH="${HOURS[iterator]}" + FILE=${PREFIX}${YYMMDD}"_"${HH}"_006.grb2" + URL="${DOMAIN}${FROM_YY}${FROM_MM}/${YYMMDD}" + + wget -c "${URL}/${FILE}" -P "grib_data" + echo "${URL}/${FILE}" >> grib_data/file_for_downloads.txt + echo "${FROM_YY}${FROM_MM}${PR_DD} ${PR_HH}00 ${FILE} ON DISC" >> AVAILABLE + done +done From 924348015fc08bef3bde158caff2d872d141ee8f Mon Sep 17 00:00:00 2001 From: Synkevych Roman Date: Fri, 26 Aug 2022 22:34:33 +0300 Subject: [PATCH 2/5] fixed url to work with historical data --- template/downloads_grib_test.sh | 52 ++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/template/downloads_grib_test.sh b/template/downloads_grib_test.sh index d7addf8..ac4ff93 100755 --- a/template/downloads_grib_test.sh +++ b/template/downloads_grib_test.sh @@ -8,13 +8,14 @@ TO_YEAR=$(date -d "$2" '+%Y'); TO_MONTH=$(date -d "$2" '+%m'); TO_DAY=$(date -d FROM_to_sec=$(date --date="$1" +%s); TO_to_sec=$(date --date="$2" +%s) AMOUNT_OF_DAYS=$(( (TO_to_sec - FROM_to_sec)/86400)) -HOURS=(0000 0600 1200 1800 0000) -URL="https://www.ncei.noaa.gov/data/global-forecast-system/access/grid-003-1.0-degree/" -PREFIX="gfs_3_" +HOUR=(0000 0600 1200 1800 0000) +SUFIX=(003 006) +URL="https://www.ncei.noaa.gov/data/global-forecast-system/access/historical/" -# if YYMM <= 7 month from current date use first forecast else analysis +# if YYMM <= 7 month from current date use first link else historical now=$(date); active_date=$(date --date="$now -7 month" +%s); date_from=$(date --date="$1" +%s) +PREFIX="gfsanl_3_" if [ $date_from -ge $active_date ]; then DOMAIN="${URL}forecast/" else @@ -23,6 +24,7 @@ fi # if folder does not exist create them [ -d grib_data ] || mkdir grib_data +DATA_DIR=$(pwd)/grib_data for i in `seq 0 $AMOUNT_OF_DAYS`; do # if day less than 10 add 0 before @@ -33,32 +35,48 @@ for i in `seq 0 $AMOUNT_OF_DAYS`; do FILE=${PREFIX}${YYMMDD}"_0000_000.grb2" URL="${DOMAIN}${FROM_YY}${FROM_MM}/${YYMMDD}" - wget -c "${URL}/${FILE}" -P "grib_data" - echo "${URL}/${FILE}" >> grib_data/file_for_downloads.txt - echo "${YYMMDD} 000000 ${FILE} ON DISC" >> AVAILABLE + if [[ -f "$DATA_DIR/${FILE}" ]]; then + echo "${FILE} exist" + else + wget_output=$(wget -c "${URL}/${FILE}" -P "grib_data") + if [ $? -ne 0 ]; then + echo "Error loading ${FILE} from ${URL}" >> grib_data/downloads.log + else + echo "${URL}/${FILE}" >> grib_data/files_for_download.txt + echo "${YYMMDD} 000000 ${FILE} ON DISC" >> AVAILABLE + fi + fi fi # set the date for the next day date=$(date --date="$1 + $i day"); FROM_YY=$(date -d "$date" '+%Y'); FROM_MM=$(date -d "$date" '+%m'); FROM_DD=$(date -d "$date" '+%d') YYMMDD="${FROM_YY}${FROM_MM}${FROM_DD}" - # download 4 forecasts for every day - for j in `seq 0 3`; do + # download 4 forecasts for every day + for j in `seq 0 7`; do # if hour 1800 increment day PR_DD=${FROM_DD} if [ "$j" -eq 3 ]; then - PR_date=$(date -d "$date + 1 day") - PR_DD=$(date -d "$PR_date" '+%d') + PR_DATE=$(date -d "$date + 1 day") + PR_DD=$(date -d "$PR_DATE" '+%d') fi - HH="${HOURS[j]}" + HH="${HOUR[j]}" iterator=$((j+1)) - PR_HH="${HOURS[iterator]}" - FILE=${PREFIX}${YYMMDD}"_"${HH}"_006.grb2" + PR_HH="${HOUR[iterator]}" + FILE=${PREFIX}${YYMMDD}"_"${HH}"_003.grb2" URL="${DOMAIN}${FROM_YY}${FROM_MM}/${YYMMDD}" - wget -c "${URL}/${FILE}" -P "grib_data" - echo "${URL}/${FILE}" >> grib_data/file_for_downloads.txt - echo "${FROM_YY}${FROM_MM}${PR_DD} ${PR_HH}00 ${FILE} ON DISC" >> AVAILABLE + if [[ -f "$DATA_DIR/${FILE}" ]]; then + echo "${FILE} exist" + else + wget_output=$(wget -c "${URL}/${FILE}" -P "grib_data") + if [ $? -ne 0 ]; then + echo "Error loading ${FILE} from ${URL}" >> grib_data/downloads.log + else + echo "${URL}/${FILE}" >> grib_data/downloads.log + echo "${FROM_YY}${FROM_MM}${PR_DD} ${PR_HH}00 ${FILE} ON DISC" >> AVAILABLE + fi + fi done done From d8a0e615f2dabc93e8b6729009f87981dc446ad7 Mon Sep 17 00:00:00 2001 From: Synkevych Roman Date: Tue, 30 Aug 2022 00:02:48 +0300 Subject: [PATCH 3/5] Fixed prefix and refactor code --- template/downloads_grib_test.sh | 36 +++++---------------------------- 1 file changed, 5 insertions(+), 31 deletions(-) diff --git a/template/downloads_grib_test.sh b/template/downloads_grib_test.sh index ac4ff93..d1c1fd1 100755 --- a/template/downloads_grib_test.sh +++ b/template/downloads_grib_test.sh @@ -8,7 +8,7 @@ TO_YEAR=$(date -d "$2" '+%Y'); TO_MONTH=$(date -d "$2" '+%m'); TO_DAY=$(date -d FROM_to_sec=$(date --date="$1" +%s); TO_to_sec=$(date --date="$2" +%s) AMOUNT_OF_DAYS=$(( (TO_to_sec - FROM_to_sec)/86400)) -HOUR=(0000 0600 1200 1800 0000) +HOUR=(0600 0600 1200 1200 1800 1800) SUFIX=(003 006) URL="https://www.ncei.noaa.gov/data/global-forecast-system/access/historical/" @@ -28,43 +28,17 @@ DATA_DIR=$(pwd)/grib_data for i in `seq 0 $AMOUNT_OF_DAYS`; do # if day less than 10 add 0 before - YYMMDD="${FROM_YY}${FROM_MM}${FROM_DD}" + YYMMDD="$(FROM_YY FROM_MM FROM_DD)" - # at first step download 0000_000 forecast - if [ "$i" -eq 0 ]; then - FILE=${PREFIX}${YYMMDD}"_0000_000.grb2" - URL="${DOMAIN}${FROM_YY}${FROM_MM}/${YYMMDD}" - - if [[ -f "$DATA_DIR/${FILE}" ]]; then - echo "${FILE} exist" - else - wget_output=$(wget -c "${URL}/${FILE}" -P "grib_data") - if [ $? -ne 0 ]; then - echo "Error loading ${FILE} from ${URL}" >> grib_data/downloads.log - else - echo "${URL}/${FILE}" >> grib_data/files_for_download.txt - echo "${YYMMDD} 000000 ${FILE} ON DISC" >> AVAILABLE - fi - fi - fi # set the date for the next day date=$(date --date="$1 + $i day"); FROM_YY=$(date -d "$date" '+%Y'); FROM_MM=$(date -d "$date" '+%m'); FROM_DD=$(date -d "$date" '+%d') - YYMMDD="${FROM_YY}${FROM_MM}${FROM_DD}" # download 4 forecasts for every day for j in `seq 0 7`; do - # if hour 1800 increment day - PR_DD=${FROM_DD} - if [ "$j" -eq 3 ]; then - PR_DATE=$(date -d "$date + 1 day") - PR_DD=$(date -d "$PR_DATE" '+%d') - fi - HH="${HOUR[j]}" - iterator=$((j+1)) - PR_HH="${HOUR[iterator]}" - FILE=${PREFIX}${YYMMDD}"_"${HH}"_003.grb2" + PR_HH="${HOUR[j]}"" + FILE=${PREFIX}${YYMMDD}"_"${HH}"_"${SUFIX[j%2]}".grb2" URL="${DOMAIN}${FROM_YY}${FROM_MM}/${YYMMDD}" if [[ -f "$DATA_DIR/${FILE}" ]]; then @@ -75,7 +49,7 @@ for i in `seq 0 $AMOUNT_OF_DAYS`; do echo "Error loading ${FILE} from ${URL}" >> grib_data/downloads.log else echo "${URL}/${FILE}" >> grib_data/downloads.log - echo "${FROM_YY}${FROM_MM}${PR_DD} ${PR_HH}00 ${FILE} ON DISC" >> AVAILABLE + echo "$(FROM_YY FROM_MM FROM_DD PR_HH)00 ${FILE} ON DISC" >> AVAILABLE fi fi done From 528513d1b791f54bc174bab09072dd46920b6140 Mon Sep 17 00:00:00 2001 From: Synkevych Roman Date: Tue, 30 Aug 2022 19:15:06 +0300 Subject: [PATCH 4/5] refactor AVAILABLE row creator --- template/downloads_grib_test.sh | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/template/downloads_grib_test.sh b/template/downloads_grib_test.sh index d1c1fd1..f8e3eb8 100755 --- a/template/downloads_grib_test.sh +++ b/template/downloads_grib_test.sh @@ -8,8 +8,9 @@ TO_YEAR=$(date -d "$2" '+%Y'); TO_MONTH=$(date -d "$2" '+%m'); TO_DAY=$(date -d FROM_to_sec=$(date --date="$1" +%s); TO_to_sec=$(date --date="$2" +%s) AMOUNT_OF_DAYS=$(( (TO_to_sec - FROM_to_sec)/86400)) -HOUR=(0600 0600 1200 1200 1800 1800) -SUFIX=(003 006) +HHMMSS=(030000 060000 090000 120000 150000 180000 210000 000000) +FILE_HOURS=(0000 0000 0600 0600 1200 1200 1800 1800) +FILE_SUFIX=(003 006) URL="https://www.ncei.noaa.gov/data/global-forecast-system/access/historical/" # if YYMM <= 7 month from current date use first link else historical @@ -26,30 +27,38 @@ fi [ -d grib_data ] || mkdir grib_data DATA_DIR=$(pwd)/grib_data +echo "YYYYMMDD HHMMSS name of the file(up to 80 characters)" >> AVAILABLE + for i in `seq 0 $AMOUNT_OF_DAYS`; do # if day less than 10 add 0 before - YYMMDD="$(FROM_YY FROM_MM FROM_DD)" + YYMMDD="${FROM_YY}${FROM_MM}${FROM_DD}" # set the date for the next day date=$(date --date="$1 + $i day"); FROM_YY=$(date -d "$date" '+%Y'); FROM_MM=$(date -d "$date" '+%m'); FROM_DD=$(date -d "$date" '+%d') + YYMMDD="${FROM_YY}${FROM_MM}${FROM_DD}" # download 4 forecasts for every day for j in `seq 0 7`; do - HH="${HOUR[j]}" - PR_HH="${HOUR[j]}"" - FILE=${PREFIX}${YYMMDD}"_"${HH}"_"${SUFIX[j%2]}".grb2" + # if hour 1800 increment day + PR_DD=${FROM_DD} + if [ "$j" -eq 7 ]; then + PR_date=$(date -d "$date + 1 day") + PR_DD=$(date -d "$PR_date" '+%d') + fi + + FILE=${PREFIX}${YYMMDD}"_"${FILE_HOURS[j]}"_"${FILE_SUFIX[j%2]}".grb2" URL="${DOMAIN}${FROM_YY}${FROM_MM}/${YYMMDD}" if [[ -f "$DATA_DIR/${FILE}" ]]; then - echo "${FILE} exist" + echo "File ${FILE} exist" else wget_output=$(wget -c "${URL}/${FILE}" -P "grib_data") if [ $? -ne 0 ]; then echo "Error loading ${FILE} from ${URL}" >> grib_data/downloads.log else - echo "${URL}/${FILE}" >> grib_data/downloads.log - echo "$(FROM_YY FROM_MM FROM_DD PR_HH)00 ${FILE} ON DISC" >> AVAILABLE + echo "File ${FILE} saved successfully" >> grib_data/downloads.log + echo "${FROM_YY}${FROM_MM}${PR_DD} ${HHMMSS[j]} ${FILE} ON DISC" >> AVAILABLE fi fi done From c3022f936464665768218827c8ceb883d50d9567 Mon Sep 17 00:00:00 2001 From: Roman Date: Mon, 5 Sep 2022 16:28:41 +0300 Subject: [PATCH 5/5] Update downloads_grib_test.sh --- template/downloads_grib_test.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/template/downloads_grib_test.sh b/template/downloads_grib_test.sh index f8e3eb8..2f8f94a 100755 --- a/template/downloads_grib_test.sh +++ b/template/downloads_grib_test.sh @@ -24,11 +24,9 @@ else fi # if folder does not exist create them -[ -d grib_data ] || mkdir grib_data +[ -d grib_data ] || mkdir grib_data && echo "YYYYMMDD HHMMSS name of the file(up to 80 characters)" > AVAILABLE DATA_DIR=$(pwd)/grib_data -echo "YYYYMMDD HHMMSS name of the file(up to 80 characters)" >> AVAILABLE - for i in `seq 0 $AMOUNT_OF_DAYS`; do # if day less than 10 add 0 before YYMMDD="${FROM_YY}${FROM_MM}${FROM_DD}"