Skip to content

Parallel Jobs

Example: Airport monitoring

  • Get a Sentinel-2 clipped image for 10 airports in a country.
  • Run all jobs in parallel
  • Visualize the results
In [1]:
import up42
import pandas as pd
import geopandas as gpd
from pathlib import Path

Random airports in Spain

In [2]:
country = "Spain"

dat = "https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat"
airports = pd.read_table(dat, 
                         sep=",", 
                         usecols=[0, 1, 3, 6, 7], 
                         names=["uid",'airport', "country", "lat", "lon"])
airports = airports[airports.country==country]
airports = gpd.GeoDataFrame(airports, geometry=gpd.points_from_xy(airports.lon, airports.lat))

world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))
world = world[world.name == country]
airports = airports[airports.within(world.iloc[0].geometry)]

display(airports.head())
uid airport country lat lon geometry
1180 1211 Albacete-Los Llanos Airport Spain 38.948502 -1.863520 POINT (-1.86352 38.94850)
1181 1212 Alicante International Airport Spain 38.282200 -0.558156 POINT (-0.55816 38.28220)
1182 1213 Almería International Airport Spain 36.843899 -2.370100 POINT (-2.37010 36.84390)
1183 1214 Asturias Airport Spain 43.563599 -6.034620 POINT (-6.03462 43.56360)
1184 1215 Córdoba Airport Spain 37.841999 -4.848880 POINT (-4.84888 37.84200)
In [3]:
airports=airports.sample(6)
In [4]:
# Visualize locations
ax = world.plot(figsize=(10,10), color='white', edgecolor='black')
airports.plot(markersize=20, ax=ax, color="r")
Out[4]:
<AxesSubplot:>
In [5]:
# Buffer airport point locations by roughly 100m
airports.geometry = airports.geometry.buffer(0.001)

Prepare UP42 workflows

Create a new project on UP42 or use an existing one.

In [6]:
# Authenticate with UP42
up42.authenticate(project_id="12345", project_api_key="6889")
#up42.authenticate(cfg_file="config.json")

project = up42.initialize_project()
2020-08-07 10:04:37,852 - up42.auth - INFO - Authentication with UP42 successful!
2020-08-07 10:04:37,853 - up42 - INFO - Working on Project with project_id 75d25f7a-426d-495f-8cfa-e54a57d2da74
In [7]:
# Increase the parallel job limit for the project.
# Only works when you have added your credit card information to the UP42 account.
project.update_project_settings(max_concurrent_jobs=10)
2020-08-07 10:04:39,248 - up42.project - INFO - Updated project settings: [{'name': 'JOB_QUERY_MAX_AOI_SIZE', 'value': '100'}, {'name': 'MAX_CONCURRENT_JOBS', 'value': '10'}, {'name': 'JOB_QUERY_LIMIT_PARAMETER_MAX_VALUE', 'value': '10'}]
In [8]:
workflow = project.create_workflow("workflow_airports", use_existing=True)
2020-08-07 10:04:39,255 - up42.project - INFO - Getting existing workflows in project ...
2020-08-07 10:04:39,627 - up42.project - INFO - Got 3 workflows for project 75d25f7a-426d-495f-8cfa-e54a57d2da74.
100%|██████████| 3/3 [00:01<00:00,  2.73it/s]
2020-08-07 10:04:40,749 - up42.project - INFO - Using existing workflow: workflow_airports - 4022cf7c-fa26-4dd4-a2c6-b37583cacc27
In [9]:
# Fill the workflow with tasks

#blocks = up42.get_blocks(basic=True)
selected_block = "sobloo-s2-l1c-aoiclipped"
workflow.add_workflow_tasks([selected_block])

workflow.get_workflow_tasks(basic=True)
2020-08-07 10:04:42,249 - up42.workflow - INFO - Added tasks to workflow: [{'name': 'sobloo-s2-l1c-aoiclipped:1', 'parentName': None, 'blockId': 'a2daaab4-196d-4226-a018-a810444dcad1'}]
2020-08-07 10:04:42,697 - up42.workflow - INFO - Got 1 tasks/blocks in workflow 4022cf7c-fa26-4dd4-a2c6-b37583cacc27.
Out[9]:
{'sobloo-s2-l1c-aoiclipped:1': '10b80b0d-9b99-4a2d-b5a4-2cbd8331cc0d'}

Run jobs in parallel

Queries & downloads one image per airport in parallel.

Crude, this will soon be available in the API in one simple command!

In [10]:
# Run jobs in parallel
up42.settings(log=False)

input_parameters_list = workflow.construct_parameters_parallel(geometries=airports.geometry.to_list(),
                                                               interval_dates=[("2018-01-01","2020-12-31")], 
                                                               geometry_operation="bbox")
# Adjust cloud cover parameter
for params in input_parameters_list:
    params[f"{selected_block}:1"]["max_cloud_cover"] = 10
2020-08-07 10:04:42,703 - up42 - INFO - Logging disabled - use up42.settings(log=True) to reactivate.
In [11]:
real_jobs = workflow.run_jobs_parallel(input_parameters_list=input_parameters_list)
In [ ]:
real_jobs.download_results()
In [13]:
# Visualize downloaded results
real_jobs.plot_results()
In [ ]: