Merge pull request #45 from asfadmin/erl/doc/update_documentation

Updated Documentation of Client and Api

README.md

@@ -2,27 +2,34 @@
 The goal of this project is to rewrite and consolidate the existing codebases for creating heatmaps of satellite data to create an interactive heatmap
 
 ## Compiling Locally
-1. Create a file named `.env` in the same directory as ingest.py
-2. `.env`  should contain login credentials to the PostgreSQL DB, ie.
+### Generate sat_data.geojson
+1. Navigate to the data-ingest directory, `cd data-ingest`
+2. Create a file named `.env` 
+3. `.env`  should contain login credentials to the PostgreSQL DB, ie.
    ```
    export DB_HOST=change_me
    export DB_USERNAME=change_me
    export DB_PASSWORD=change_me
    export DB_NAME=change_me
    ```
-3.  1)  If you have the dependencies installed locally you can now run `python3 ingest.py` and `sat_data.geojson` will be generated
+4.  1)  If you have the dependencies installed locally you can now run `python3 ingest.py` and `sat_data.geojson` will be generated
 
     2)  If you have conda installed then you can create a conda enviornment using `env.yml` inside the `Docker` directory, you can then run `python3 ingest.py` inside this environment to generate `sat_data.geojson`
-
-    3)  If you have docker installed then you can `cd` into `Docker` and then enter `./run.sh` which will generate `sat_data.geojson` 
 
+### Setting up rust
+1. Install rust, rust-lang.org is the page you're looking for
+2. This project uses nightly features of rust, this means you will need a nightly version of rust, run `rustup toolchain install nightly`
+3. To swtich to a nightly build of rust run `rustup override set nightly`
 
-## Dependancies
-- postgis
-- shapely
-- pandas
-- geopandas
-- matplotlib
+### Setting up the server
+1. Move `sat_data.geojson` to the `heatmap-service` directory, don't change the file name or the server will fail to find the data
+2. Navigate into the `heatmap-service` directory, `cd heatmap-service`
+3. Run `cargo run` in the terminal and you now have a locally running version of the server, if the terminal you entered this command into closes you will need to repeat this step in a new terminal
+
+### Setting up the client
+1. Navigate to the `heatmap-client` directory, `cd heatmap-client`
+2. Install trunk, run `cargo binstall trunk`
+3. Run `trunk serve --open`, this should open a page in your default browser, if you would prefer the command not open a page remove `--open` and it will serve the client without opening a new page
 
 ## Contributing
 Elliott Lewandowski

data-ingest/README.md

@@ -1,3 +1,25 @@
 # Data Ingest
 
-This directory handles pulling data from a PostgreSQL database and formatting. The file it generates is served to the heatmap-client from the heatmap-service
+This directory handles pulling data from a PostgreSQL database and formatting. The file it generates is served to the heatmap-client from the heatmap-service
+
+## Compiling Locally
+
+1. Create a file named `.env` 
+2. `.env`  should contain login credentials to the PostgreSQL DB, ie.
+   ```
+   export DB_HOST=change_me
+   export DB_USERNAME=change_me
+   export DB_PASSWORD=change_me
+   export DB_NAME=change_me
+   ```
+3.  1)  If you have the dependencies installed locally you can now run `python3 ingest.py` and `sat_data.geojson` will be generated
+
+    2)  If you have conda installed then you can create a conda enviornment using `env.yml` inside the `Docker` directory, you can then run `python3 ingest.py` inside this environment to generate `sat_data.geojson`
+
+
+## Dependancies
+- postgis
+- shapely
+- pandas
+- geopandas
+- dotenv

data-ingest/create_sql.py

@@ -13,22 +13,22 @@
 
 def generate_command(
     start=date.datetime(2021, 1, 1),
-    end=date.datetime(2021, 2, 1),
+    end=date.datetime(2021, 1, 10),
     platform_type="'SA', 'SB'",
     data_type="'GRD'",
 ):
 
     cmd = (
         """SELECT ST_AsText(ST_Centroid(shape)), g.granule_name, g.platform_type, g.data_sensor_type, g.start_time, g.shape
-        
-                FROM granule g 
-                
+
+                FROM granule g
+
                 where g.platform_type in ("""
         + platform_type
         + """) and
-                
+
                 g.data_granule_type in ('SENTINEL_1A_FRAME', 'SENTINEL_1B_FRAME'  ) and
-                
+
                 substr(granule_name, 8, 3) in ("""
         + data_type
         + """) and
@@ -39,8 +39,8 @@ def generate_command(
         + start.strftime("%x")
         + """' and '"""
         + end.strftime("%x")
-        + """'    
-            
+        + """'
+
                 order by shape asc;"""
     )
 

heatmap-api/README.md

@@ -0,0 +1,3 @@
+# Heatmap API
+
+This directory contains code that is used by both `heatmap-service` and `heatmap-client`. It exists to reduce code duplication. It mainly contains code related to the structure of data requests and responses.

heatmap-api/src/lib.rs

@@ -4,6 +4,7 @@ pub trait ToPartialString {
     fn _to_partial_string(&self) -> String;
 }
 
+// Enums defining possible filter options
 #[derive(Deserialize, Serialize, Clone, Copy, Debug, PartialEq)]
 pub enum ProductTypes {
     #[serde(rename = "GRD")]
@@ -58,6 +59,7 @@ impl DataSensor {
     }
 }
 
+// The filter passed from client to server on a request for data
 #[derive(Deserialize, Serialize, Clone)]
 pub struct Filter {
     pub product_type: Vec<ProductTypes>,
@@ -66,11 +68,19 @@ pub struct Filter {
     pub end_date: String,
 }
 
+// Client sends this to server
 #[derive(Deserialize, Serialize)]
 pub struct HeatmapQuery {
     pub filter: Filter,
 }
 
+// Server sends this back to client after a query,
+// contains the granule data
+#[derive(Deserialize, Serialize, Debug, PartialEq)]
+pub struct HeatmapResponse {
+    pub data: HeatmapData,
+}
+
 #[derive(Deserialize, Serialize, Debug, PartialEq)]
 pub struct HeatmapData {
     pub data: InteriorData,
@@ -82,18 +92,14 @@ pub struct InteriorData {
     pub weights: Vec<u64>,
 }
 
+// Server sends this back to client after a query,
+// contains world outline data
 #[derive(Deserialize, Serialize, Debug, PartialEq)]
-pub struct HeatmapResponse {
-    pub data: HeatmapData,
+pub struct OutlineResponse {
+    pub data: OutlineData,
 }
-
 #[derive(Deserialize, Serialize, Debug, PartialEq)]
 pub struct OutlineData {
     pub length: i32,
     pub positions: Vec<Vec<(f64, f64)>>,
 }
-
-#[derive(Deserialize, Serialize, Debug, PartialEq)]
-pub struct OutlineResponse {
-    pub data: OutlineData,
-}

heatmap-client/README.md

@@ -2,8 +2,17 @@
 
 This code is responsible for the client side of the heatmap generation process. 
 
-The heavy lifting of generating the actual heatmap occurs in src/canvas
+`./src/canvas` does the heavy lifting of generating the actual heatmap 
 
     The GPU is leveraged to generate these heatmaps, a large portion of the code is getting wgpu to play nicley in wasm
 
-The ingest folder requests data from the server located in the heatmap-service directory
+`./src/ingest` requests data from the server located in the heatmap-service directory
+
+`./src/ui` This contains the user interface that is overlayed onto the heatmap
+
+`./assets` contains static assets used in the client, ie. colormap textures and resources to export a png
+
+## Compiling Locally
+1. Ensure rust is on a nightly build, you can check with `rustup toolchain list`
+2. Install trunk, run `cargo binstall trunk`
+3. Run `trunk serve --open`, this should open a page in your default browser, if you would prefer the command not open a page remove `--open` and it will serve the client without opening a new page

heatmap-client/src/canvas/app.rs

@@ -27,10 +27,13 @@ pub struct App<'a> {
 }
 
 // The application handler instance doesnt allow for error handling
+// The application handler responds to changes in the event loop, we send custom events here using
+//    an event_loop_proxy
 
 impl<'a> ApplicationHandler<UserMessage<'static>> for App<'a> {
+    // This is run on initial startup, creates a window and stores it in the state, also stores
+    //     the windows canvas in external state
     fn resumed(&mut self, event_loop: &ActiveEventLoop) {
-        // Create the window and add it to state
         self.state.window = Some(Rc::new(
             event_loop
                 .create_window(
@@ -40,6 +43,7 @@ impl<'a> ApplicationHandler<UserMessage<'static>> for App<'a> {
                 .expect("ERROR: Failed to create window"),
         ));
 
+        // Convert web_sys HtmlCanvasElement into a leptos HtmlElement<AnyElement>
         self.external_state.borrow_mut().canvas = self
             .state
             .window
@@ -110,7 +114,7 @@ impl<'a> ApplicationHandler<UserMessage<'static>> for App<'a> {
             }
 
             WindowEvent::Resized(physical_size) => {
-                // Initialize setup of state when resized is first called
+                // Initialize setup of state when resized is first called, otherwise call state.resize
                 if self.state.init_stage == InitStage::Incomplete {
                     web_sys::console::log_1(&"Generating state...".into());
                     leptos::spawn_local(super::render_context::generate_render_context(
@@ -127,6 +131,7 @@ impl<'a> ApplicationHandler<UserMessage<'static>> for App<'a> {
                 }
             }
 
+            // Any other event will be handled by the user input handler
             _ => {
                 self.state.handle_input_event(event);
             }
@@ -161,6 +166,7 @@ impl<'a> ApplicationHandler<UserMessage<'static>> for App<'a> {
                 );
             }
 
+            // There is incoming data from the service, we need to place this new data into buffers to render
             UserMessage::IncomingData(data, outline_data) => {
                 web_sys::console::log_1(&"Generating Buffers...".into());
                 let render_context = self
@@ -190,29 +196,36 @@ impl<'a> ApplicationHandler<UserMessage<'static>> for App<'a> {
 
                 web_sys::console::log_1(&"Done Generating Buffers".into());
 
+                // Turn off the loading wheel
                 self.external_state.borrow_mut().set_ready.set(true);
             }
 
+            // This is part of getting the max weight of a set of data, to get data from the GPU
+            //    you have to map a buffer to the CPU, this is done asynchronously so we fire off
+            //    a custom event on mapping completion
             UserMessage::BufferMapped => {
                 let render_context = self
                     .state
                     .render_context
                     .as_mut()
                     .expect("Failed to get render context in UserMessage::BufferMapped");
 
+                // We read the data contained in the buffer and convert it from &[u8] to Vec<u8>
                 let raw_bytes: Vec<u8> = (&*render_context
                     .max_weight_context
                     .buffer
                     .slice(..)
                     .get_mapped_range())
                     .into();
-                let mut red_data: Vec<f32> = Vec::new();
 
+                // The buffer is formated for f32 but we pulled a Vec<u8>, we must reform the Vec<f32> from the bytes
+                let mut red_data: Vec<f32> = Vec::new();
                 let mut raw_iter = raw_bytes.iter();
-
                 while let Ok(raw) = raw_iter.next_chunk::<4>() {
+                    // Read one channel into a f32
                     red_data.push(f32::from_le_bytes([*raw[0], *raw[1], *raw[2], *raw[3]]));
 
+                    // The texture we stored in the buffer was rgba32Float but only had red data so we skip the g, b, a channels
                     match raw_iter.advance_by(4 * 3) {
                         Ok(_) => {}
                         Err(_) => {
@@ -221,6 +234,7 @@ impl<'a> ApplicationHandler<UserMessage<'static>> for App<'a> {
                     }
                 }
 
+                // Find the max value in the Vec<f32> we just created
                 let mut max = 0.0;
 
                 for value in red_data.iter() {
@@ -231,9 +245,11 @@ impl<'a> ApplicationHandler<UserMessage<'static>> for App<'a> {
 
                 web_sys::console::log_1(&format!("Max: {:?}", max).into());
 
+                // We now update the uniform buffer with our max weight
+                //    so that we can read the max value in the colormap render pass
                 let mut uniform_data: Vec<u8> = max.to_le_bytes().into();
 
-                // Uniform Buffer must be 16 byte aligned
+                // Uniform Buffer must be 16 byte aligned so we pad it with 0's
                 while uniform_data.len() % 16 != 0 {
                     uniform_data.push(0);
                 }