-
Notifications
You must be signed in to change notification settings - Fork 49
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Add script to run tests that will * optionally run in both local and ray mode * split between fast and slow tests * verify all notebooks are accounted * Fix file scans so they can run with invalid AWS credentials in environment, same trick as for materialize, try being anonymous if default doesn't work. * Fix ndd_example notebook to not specify parallelism. * Fix tutorial notebook to use materialize to avoid some re-execution * add some logging * lock in the new 0.1.24 sycamore release * ignore notebooks/tmp
- Loading branch information
1 parent
f6c47fd
commit f0a358c
Showing
10 changed files
with
265 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -47,3 +47,4 @@ notebooks/default-prep-data | |
| .python-version | ||
| apps/timetrace/ttviz | ||
| notebooks/pc-tutorial | ||
| notebooks/tmp | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| #!/bin/bash | ||
| main() { | ||
| config | ||
| cd notebooks | ||
| if [[ "$1" == --fast ]]; then | ||
| echo "Testing fast notebooks..." | ||
| local start=$(date +%s) | ||
| test_notebooks "${FAST_NOTEBOOKS[@]}" | ||
| local end=$(date +%s) | ||
| local elapsed=$(expr $end - $start) | ||
| echo "-------------------------------------------------------" | ||
| echo "--- Testing fast notebooks took $elapsed/$FAST_MAXTIME seconds" | ||
| if [[ $elapsed -ge $FAST_MAXTIME ]]; then | ||
| echo "--- Fast tests took too long." | ||
| echo "--- Move some fast tests to slow," | ||
| echo "--- or speed up the fast notebooks." | ||
| exit 1 | ||
| fi | ||
| echo "-------------------------------------------------------" | ||
| check_coverage | ||
| elif [[ "$1" == --slow ]]; then | ||
| echo "Testing slow notebooks..." | ||
| test_notebooks "${SLOW_NOTEBOOKS[@]}" | ||
| else | ||
| echo "Usage: $0 {--fast,--slow}" | ||
| exit 1 | ||
| fi | ||
| exit 0 | ||
| } | ||
|
|
||
| config() { | ||
| FAST_NOTEBOOKS=( | ||
| default-prep-script.ipynb # 20+40s on Eric's laptop. | ||
| OpenAI-logprob.ipynb # 5s on Eric's laptop. | ||
| ) | ||
|
|
||
| FAST_MAXTIME=120 | ||
| SLOW_NOTEBOOKS=( | ||
| ArynPartitionerExample.ipynb # Gets a crash message, but passes? | ||
| ArynPartitionerPython.ipynb | ||
| jupyter_dev_example.ipynb # O(n^2) from re-show/take | ||
| metadata-extraction.ipynb # processes entire | ||
| sycamore_demo.ipynb # O(n^2) from re-show/take | ||
| tutorial.ipynb # aryn partitioner on entire sort benchmark | ||
| VisualizePartitioner.ipynb | ||
| ) | ||
| EXCLUDE_NOTEBOOKS=( | ||
| # No good reason for exclusion, just what was | ||
| # not automatically tested before | ||
| ArynPartitionerWithLangchain.ipynb # needs langchain to be installed | ||
| duckdb-writer.ipynb # timeout | ||
| elasticsearch-writer.ipynb # needs elasticsearch db setup | ||
| financial-docs-10k-example.ipynb # needs to be fixed like | ||
| # tutorial to have assertion, assertion code needs to move to aryn_sdk | ||
| ndd_example.ipynb #broken | ||
| ntsb-demo.ipynb # needs rps | ||
| opensearch_docs_etl.ipynb # timeout | ||
| opensearch-writer.ipynb # depends on langchain | ||
| pinecone-writer.ipynb # needs pinccone key | ||
| query-demo.ipynb # depnds on un-checked in visualize.py | ||
| subtask-sample.ipynb # broken -- old style llm prompts | ||
| sycamore-tutorial-intermediate-etl.ipynb # timeout | ||
| unpickle_query.ipynb # looking for uncommitted file | ||
| weaviate-writer.ipynb # path not set to files | ||
| ) | ||
| } | ||
|
|
||
| test_notebooks() { | ||
| for i in "$@"; do | ||
| echo | ||
| echo | ||
| echo "-------------------------------------------------------------------------" | ||
| echo "Starting test on $i as written" | ||
| echo "-------------------------------------------------------------------------" | ||
|
|
||
| time poetry run pytest --nbmake --nbmake-timeout=600 $i || exit 1 | ||
|
|
||
| if [[ $(grep -c sycamore.EXEC_LOCAL $i) -ge 1 ]]; then | ||
| sed -e 's/sycamore.EXEC_LOCAL/sycamore.EXEC_RAY/' <$i >ray-variant-$i | ||
| echo "-------------------------------------------------------------------------" | ||
| echo "Starting test on $i with EXEC_RAY" | ||
| echo "-------------------------------------------------------------------------" | ||
| time poetry run pytest --nbmake --nbmake-timeout=600 ray-variant-$i || exit 1 | ||
| rm ray-variant-$i | ||
| fi | ||
| done | ||
| } | ||
|
|
||
| check_coverage() { | ||
| echo "Verifying coverage of all notebooks..." | ||
| ls *.ipynb | grep -v '^ray-variant-' >/tmp/notebooks.list | ||
| ( | ||
| for i in "${FAST_NOTEBOOKS[@]}" "${SLOW_NOTEBOOKS[@]}" "${EXCLUDE_NOTEBOOKS[@]}"; do | ||
| echo "$i" | ||
| done | ||
| ) >>/tmp/notebooks.list | ||
| sort /tmp/notebooks.list | uniq -c | sort -n | grep -v '^ 2 ' >/tmp/notebooks.unlisted | ||
| if [[ $(wc -l </tmp/notebooks.unlisted) != 0 ]]; then | ||
| echo "ERROR: some notebooks are unlisted." | ||
| echo " Add them to FAST_NOTEBOOKS for fast tests that block commits," | ||
| echo " SLOW_NOTEBOOKS for slower tests that do not block commits," | ||
| echo " or EXCLUDE_NOTEBOOKS for notebooks that should not be automatically tested." | ||
| echo " Missing noteboos:" | ||
| sed 's/^ 1 //' /tmp/notebooks.unlisted | ||
| exit 1 | ||
| fi | ||
| rm /tmp/notebooks.list /tmp/notebooks.unlisted | ||
| echo "All notebooks are classified as fast, slow or excluded." | ||
| } | ||
|
|
||
| main "$@" | ||
| exit 1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.