Part 11. Utilizing Cloud Storage for Image Uploads (Paid Service)

Benefits: Ability to batch upload images and automatically set properties

11.1 Retrieve Local File List

import os

# Replace with your cloud bucket and folder
#gcs_root_dir  = "gs://Bucket/folder/"
gcs_root_dir  = "gs://gis/train/"

# Local root directory
loc_root_dir = "./local_path_to_images/"

# Get the file list
tif_files = os.listdir(loc_root_dir)
print(tif_files)

11.2 Filter for TIFF Files

# Filter for TIFF files
tif_files = [f for f in tif_files if f[-3:] == "tif"]
print(tif_files)

11.3 Define Image Properties

# Define the year of the images
year = 2021

# Define the acquisition start time in milliseconds since epoch
systs = ee.Date.fromYMD(2021, 7, 1).millis().getInfo()

# Define band names for the images
bands = "HH,HV,Date,Linci,Mask"

# Debug with the first file
curf = tif_files[0]

# Extract longitude and latitude from the filename (adjust based on your naming convention)
lon = int(curf[curf.find("E")+1:curf.find("_")])
lat = int(curf[1:curf.find("E")])
grid = curf[:curf.find("_")]
print(curf, lon, lat, grid)

# Generate parameters for the upload command
asset_dir = 'projects/m1-gis-project/assets/CO2/'

11.4 Test Single Image Upload

asset_id = asset_dir + curf[:-4]
gcs_file_id = gcs_root_dir + curf

# Define properties
prop_year = '(number)Year=' + str(year)
prop_grid = '(string)Grid=' + grid
prop_long = '(number)Longitude=' + str(lon)
prop_lati = '(number)Latitude=' + str(lat)
print(asset_id, gcs_file_id, systs, bands, prop_grid, prop_year, prop_long, prop_lati)

# Execute the upload command
!earthengine upload image --asset_id=$asset_id $gcs_file_id --time_start $systs --bands $bands -p $prop_grid -p $prop_long -p $prop_lati -p $prop_year

11.5 Batch Upload

for curf in tif_files:
    asset_id = asset_dir + curf[:-4]
    gcs_file_id = gcs_root_dir + curf
    prop_year = '(number)Year=' + str(year)
    prop_grid = '(string)Grid=' + grid
    prop_long = '(number)Longitude=' + str(lon)
    prop_lati = '(number)Latitude=' + str(lat)
    print('Uploading ', curf, '\n')
    !earthengine upload image --asset_id=$asset_id $gcs_file_id --time_start $systs --bands $bands -p $prop_grid -p $prop_long -p $prop_lati -p $prop_year

4o