Feature: validation column config

dash_tabulator/src/lib/components/DashTabulator.react.js

@@ -14,7 +14,7 @@ import { extractFilterValues } from '../helpers/dashboardFilters.js'
 
 const crypto = require('crypto');
 
-const plugin_version = "2.0.8";
+const plugin_version = "2.1.0";
 
 function md5(string) {
     return crypto.createHash('md5').update(string).digest('hex');
@@ -73,10 +73,15 @@ export default class DashTabulator extends React.Component {
             edited.value = cell.getValue()
             edited.row = cell.getData()
             this.props.setProps({ cellEdited: edited })
+            if (edited.field == "_visual_edit_validated" | edited.field == "_visual_edit_notes") {
+                column_name_hash = edited.field
+            } else {
+                column_name_hash = md5(edited.field)
+            }
             try {
                 window.parent.WT1SVC.event("visualedit-edit-cell", {
                     "dataset_name_hash": md5(datasetName),
-                    "column_name_hash": md5(edited.field),
+                    "column_name_hash": column_name_hash,
                     "column_type": edited.type,
                     "plugin_version": plugin_version
                 });
@@ -98,24 +103,24 @@ export default class DashTabulator extends React.Component {
     handleFilterEvent = (event) => {
         const data = event.data;
         if (!data || data.type !== 'filters') return;
-    
+
         const filters = data.filters;
         if (filters.length === 0) {
             this.tabulator.clearFilter();
             return;
         }
-    
+
         const filter = filters[0];
         if (!filter.active || filter.filterType !== 'ALPHANUM_FACET') {
             this.tabulator.clearFilter();
             return;
         }
-    
+
         const columnFields = this.props.columns.map(col => col.field);
         if (!columnFields.includes(filter.column)) {
             return;
         }
-    
+
         const { includedValues, excludedValues } = extractFilterValues(filter);
         this.applyTableFilter(filter, includedValues, excludedValues);
     }

docs/dash-examples.md

@@ -25,7 +25,7 @@ tabulator_utils = dataiku.import_from_plugin("visual-edit", "tabulator_utils")
 dash_tabulator = dataiku.import_from_plugin("visual-edit", "dash_tabulator")
 ```
 
-Instantiate `DataEditor`, which will later be used to define the data to load into the table:
+Instantiate `DataEditor`, which will be used to get the edited data and load it when rendering the table, and to persist edits:
 
 ```python
 ORIGINAL_DATASET = "orders"

docs/editschema.md

@@ -6,26 +6,26 @@ The editschema can be used to add specific UI behavior to editable columns. It i
 
 For each column of the original dataset you can set the following fields:
 
-* `name`: name of a column of the original dataset.
-* `type`: allows to override the column's type inferred from the original dataset's schema, or to refine it:
-  * `boolean_tick` for Boolean columns: formatting and editing of boolean values behaves like a toggle with only 2 states: `true` represented by a green tick icon, and no icon at all otherwise. ![](boolean_tick.gif)
-    * Note: the default behavior of the "boolean" type uses 3 states: `true` represented by a green tick icon, `false` represented by a red cross, and empty.
-  * `textarea` for Text columns: this is ideal when values are longer bits of text and provides the following benefits:
-    * Shows text with carriage returns intact (great for multiline text).
-    * Adjusts the height of rows to fit cell contents when columns are resized.
-    * Allows entering of multiple lines of plain text (when the column is editable).
+- `name`: name of a column of the original dataset.
+- `type`: allows to override the column's type inferred from the original dataset's schema, or to refine it:
+  - `boolean_tick` for Boolean columns: formatting and editing of boolean values behaves like a toggle with only 2 states: `true` represented by a green tick icon, and no icon at all otherwise. ![](boolean_tick.gif)
+    - Note: the default behavior of the "boolean" type uses 3 states: `true` represented by a green tick icon, `false` represented by a red cross, and empty.
+  - `textarea` for Text columns: this is ideal when values are longer bits of text and provides the following benefits:
+    - Shows text with carriage returns intact (great for multiline text).
+    - Adjusts the height of rows to fit cell contents when columns are resized.
+    - Allows entering of multiple lines of plain text (when the column is editable).
 
 ## Example
 
 ```json
 [
   {
-    "name": "validated",
+    "name": "Tick column",
     "type": "boolean_tick"
   },
   {
-    "name": "notes",
+    "name": "Long text column",
     "type": "textarea"
-  },
+  }
 ]
 ```

docs/validate.md

@@ -8,29 +8,34 @@ We want business users to validate and make corrections as needed, based on thei
 
 Machine-generated data would be stored in the output dataset of an existing data pipeline. Each row would correspond to an item to validate. Columns would include:
 
-* primary keys;
-* machine-generated columns, whose values would change if the pipeline or its algorithms change;
-* display-only columns, whose values would help the end-user figure out how to validate/edit/provide feedback.
+- primary keys;
+- machine-generated columns, whose values would change if the pipeline or its algorithms change;
+- display-only columns, whose values would help the end-user figure out how to validate/edit/provide feedback.
 
-Instead of exporting this dataset to Excel, we want end-users to access a web interface to validate and correct the data. In addition to the above columns, we would want 2 feedback columns: one to mark rows as valid (via checkboxes) and one to write comments.
+Instead of exporting this dataset to Excel, we want end-users to access a web interface to validate and correct the data. In addition to the above columns, we would want 2 feedback columns: one to mark rows as valid (via checkboxes) and one to write notes.
 
 ## Special behavior of the validation column
 
 Validation columns are used to indicate that a human saw what the machine did for a given row, and had the opportunity to make corrections or to fill in missing values if needed.
 
-The webapp’s backend implements special behavior when a cell from a column named "Validated" or "Reviewed" is edited: values of all editable columns from the same row are logged (even if they weren’t edited). This allows the _editlog_ to include not just the information that the row is valid, but also to record the actual values that were validated. This is particularly useful when those values were generated by an algorithm, because they may change if the algorithm changes.
+The webapp’s backend implements special behavior when the validation column is ticked: values of all editable columns from the same row are logged, even if they weren’t edited. This allows the _editlog_ to include not just the information that the row is valid, but also to record the actual values that were validated. This is particularly useful when those values were generated by an algorithm, because they may change if the algorithm changes.
 
 As a result, there will be no missing value in the machine-generated and human-reviewed columns that are present in the _edits_ dataset, for rows marked as valid.
 
 ## How-to
 
 You must be familiar with the initial [How to Use guide](https://www.dataiku.com/product/plugins/visual-edit/#how-to-use) before following the steps below.
 
-* **Add feedback columns to the dataset to review**: this can be done via code in the existing data pipeline, or with an additional Prepare recipe, as columns with missing values to serve as placeholders in the webapp.
-* **When creating a Visual Edit webapp**: make sure to select all machine-generated columns and feedback columns as editable.
-* **When using the webapp**: you would review values in generated columns (mark as valid, or edit values and add notes when necessary) and fill in missing values.
+- **When creating a Visual Edit webapp**:
+  - Make sure to select all machine-generated columns as editable.
+  - Indicate that a validation column is required, and choose a display name for it (e.g. "Validated" or "Reviewed"). This will be used to mark rows as valid. Behind the scenes, values will be stored in a boolean column named `validated`.
+  - Do the same for a notes column, if needed. Behind the scenes, values will be stored in a string column named `notes`.
+- **When using the webapp**:
+  - Review values in generated columns and edit where necessary (e.g. to correct a value that was generated by an algorithm, or to fill in a missing value).
+  - Mark rows as valid.
+  - Write notes where useful.
 
 ## Next
 
-* [Build a complete application to test with end-users](build-complete-application)
-* [Deploy to production](deploy): this will require collaboration and testing with IT, to propagate the contents of the _edits_ dataset to other systems (columns of this dataset include primary keys, human-reviewed values, a boolean validation column, and additional human feedback columns).
+- [Build a complete application to test with end-users](build-complete-application)
+- [Deploy to production](deploy): this will require collaboration and testing with IT, to propagate the contents of the _edits_ dataset to other systems (columns of this dataset include primary keys, human-reviewed values, a boolean validation column, and additional human feedback columns).

dss-plugin-visual-edit/.vscode/settings.json

@@ -5,8 +5,8 @@
     "python.linting.enabled": true,
     "python.linting.pylintEnabled": true,
     "python.linting.flake8Enabled": false,
-    "python.pythonPath": "$./venv/bin/python",
-    "python.defaultInterpreterPath": "./venv/bin/python",
+    "python.pythonPath": "$(pyenv which python)",
+    "python.defaultInterpreterPath": "$(pyenv which python)",
     "cSpell.words": [
         "editlog"
     ],