Simplify workflow for default input

Author: pramitchoudhary

pyproject.toml

@@ -21,7 +21,7 @@ classifiers = [
 packages = [{include = "sidekick"}]
 
 [tool.poetry.dependencies]
-python = ">=3.8.1,<=3.10"
+python = ">=3.8.1,<=3.11"
 pandas = "^1.3.3"
 numpy = "^1.21.2"
 click = "^8.0.1"

requirements.txt

@@ -1,89 +1,89 @@
-accelerate==0.21.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-aiohttp==3.8.5 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-aiosignal==1.3.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-ansicon==1.89.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0" and platform_system == "Windows"
-anyio==3.7.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-async-timeout==4.0.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-attrs==23.1.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-bitsandbytes==0.41.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-blessed==1.20.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-cachetools==5.3.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-certifi==2023.7.22 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-charset-normalizer==3.2.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-click==8.1.6 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-colorama==0.4.6 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-dataclasses-json==0.5.14 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-exceptiongroup==1.1.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-filelock==3.12.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-frozenlist==1.4.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-fsspec==2023.6.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-gptcache==0.1.39.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-greenlet==2.0.2 ; python_full_version >= "3.8.1" and platform_machine == "aarch64" and python_full_version <= "3.10.0" or python_full_version >= "3.8.1" and platform_machine == "ppc64le" and python_full_version <= "3.10.0" or python_full_version >= "3.8.1" and platform_machine == "x86_64" and python_full_version <= "3.10.0" or python_full_version >= "3.8.1" and platform_machine == "amd64" and python_full_version <= "3.10.0" or python_full_version >= "3.8.1" and platform_machine == "AMD64" and python_full_version <= "3.10.0" or python_full_version >= "3.8.1" and platform_machine == "win32" and python_full_version <= "3.10.0" or python_full_version >= "3.8.1" and platform_machine == "WIN32" and python_full_version <= "3.10.0"
-h11==0.14.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-h2o-wave==0.26.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-httpcore==0.17.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-httpx==0.24.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-huggingface-hub==0.16.4 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-idna==3.4 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-inquirer==3.1.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-instructorembedding==1.0.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-jinja2==3.1.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-jinxed==1.2.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0" and platform_system == "Windows"
-joblib==1.3.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-langchain==0.0.142 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-llama-index==0.5.27 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-loguru==0.7.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-markupsafe==2.1.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-marshmallow==3.20.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-mpmath==1.3.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-multidict==6.0.4 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-mypy-extensions==1.0.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-networkx==3.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-nltk==3.8.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-numexpr==2.8.5 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-numpy==1.24.4 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-openai==0.27.8 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-openapi-schema-pydantic==1.2.4 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-packaging==23.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-pandas==1.5.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-pandasql==0.7.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-pillow==10.0.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-psutil==5.9.5 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-psycopg2-binary==2.9.7 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-pydantic==1.10.12 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-python-dateutil==2.8.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-python-editor==1.0.4 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-pytz==2023.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-pyyaml==6.0.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-readchar==4.0.5 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-regex==2023.8.8 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-requests==2.31.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-safetensors==0.3.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-scikit-learn==1.3.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-scipy==1.10.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-sentence-transformers==2.2.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-sentencepiece==0.1.99 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-setuptools==68.1.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-six==1.16.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-sniffio==1.3.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-sqlalchemy-utils==0.41.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-sqlalchemy==1.4.49 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-sqlglot==12.4.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-starlette==0.31.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-sympy==1.12 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-tenacity==8.2.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-threadpoolctl==3.2.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-tiktoken==0.4.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-tokenizers==0.13.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-toml==0.10.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-torch==2.0.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-torchvision==0.15.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-tqdm==4.66.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-transformers==4.31.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-typing-extensions==4.7.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-typing-inspect==0.9.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-urllib3==2.0.4 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-uvicorn==0.23.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-wcwidth==0.2.6 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
-win32-setctime==1.1.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0" and sys_platform == "win32"
-yarl==1.9.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.10.0"
+accelerate==0.21.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+aiohttp==3.8.5 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+aiosignal==1.3.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+ansicon==1.89.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0" and platform_system == "Windows"
+anyio==4.0.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+async-timeout==4.0.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+attrs==23.1.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+bitsandbytes==0.41.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+blessed==1.20.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+cachetools==5.3.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+certifi==2023.7.22 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+charset-normalizer==3.3.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+click==8.1.7 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+colorama==0.4.6 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+dataclasses-json==0.5.14 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+exceptiongroup==1.1.3 ; python_full_version >= "3.8.1" and python_version < "3.11"
+filelock==3.12.4 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+frozenlist==1.4.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+fsspec==2023.9.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+gptcache==0.1.42 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+greenlet==3.0.0 ; python_full_version >= "3.8.1" and platform_machine == "aarch64" and python_full_version <= "3.11.0" or python_full_version >= "3.8.1" and platform_machine == "ppc64le" and python_full_version <= "3.11.0" or python_full_version >= "3.8.1" and platform_machine == "x86_64" and python_full_version <= "3.11.0" or python_full_version >= "3.8.1" and platform_machine == "amd64" and python_full_version <= "3.11.0" or python_full_version >= "3.8.1" and platform_machine == "AMD64" and python_full_version <= "3.11.0" or python_full_version >= "3.8.1" and platform_machine == "win32" and python_full_version <= "3.11.0" or python_full_version >= "3.8.1" and platform_machine == "WIN32" and python_full_version <= "3.11.0"
+h11==0.14.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+h2o-wave==0.26.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+httpcore==0.18.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+httpx==0.25.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+huggingface-hub==0.17.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+idna==3.4 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+inquirer==3.1.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+instructorembedding==1.0.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+jinja2==3.1.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+jinxed==1.2.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0" and platform_system == "Windows"
+joblib==1.3.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+langchain==0.0.142 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+llama-index==0.5.27 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+loguru==0.7.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+markupsafe==2.1.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+marshmallow==3.20.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+mpmath==1.3.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+multidict==6.0.4 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+mypy-extensions==1.0.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+networkx==3.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+nltk==3.8.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+numexpr==2.8.6 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+numpy==1.24.4 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+openai==0.28.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+openapi-schema-pydantic==1.2.4 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+packaging==23.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+pandas==1.5.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+pandasql==0.7.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+pillow==10.0.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+psutil==5.9.5 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+psycopg2-binary==2.9.9 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+pydantic==1.10.13 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+python-dateutil==2.8.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+python-editor==1.0.4 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+pytz==2023.3.post1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+pyyaml==6.0.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+readchar==4.0.5 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+regex==2023.10.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+requests==2.31.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+safetensors==0.4.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+scikit-learn==1.3.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+scipy==1.10.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+sentence-transformers==2.2.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+sentencepiece==0.1.99 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+setuptools==68.2.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+six==1.16.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+sniffio==1.3.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+sqlalchemy-utils==0.41.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+sqlalchemy==1.4.49 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+sqlglot==12.4.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+starlette==0.31.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+sympy==1.12 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+tenacity==8.2.3 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+threadpoolctl==3.2.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+tiktoken==0.5.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+tokenizers==0.14.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+toml==0.10.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+torch==2.1.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+torchvision==0.16.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+tqdm==4.66.1 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+transformers==4.34.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+typing-extensions==4.8.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+typing-inspect==0.9.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+urllib3==2.0.6 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+uvicorn==0.23.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+wcwidth==0.2.8 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"
+win32-setctime==1.1.0 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0" and sys_platform == "win32"
+yarl==1.9.2 ; python_full_version >= "3.8.1" and python_full_version <= "3.11.0"

sidekick/db_config.py

@@ -166,19 +166,20 @@ def has_table(self):
     def add_samples(self, data_csv_path=None):
         conn_str = self._url
         try:
-            df_chunks = pd.read_csv(data_csv_path, chunksize=10000)
+            logger.debug(f"Adding sample values to table: {data_csv_path}")
+            df_chunks = pd.read_csv(data_csv_path, chunksize=5000)
             engine = create_engine(conn_str, isolation_level="AUTOCOMMIT")
 
-            sample_query = f"SELECT COUNT(*) AS ROWS FROM {self.table_name} LIMIT 1"
             for idx, chunk in enumerate(df_chunks):
                 # Write rows to database
                 logger.debug(f"Inserting chunk: {idx}")
                 chunk.columns = self.column_names
                 # Make sure column names in the data-frame match the schema
-                chunk.to_sql(self.table_name, engine, if_exists="append", index=False, method="multi")
+                chunk.to_sql(self.table_name, engine, if_exists="replace", index=False, method="multi")
 
             logger.info(f"Data inserted into table: {self.table_name}")
             # Fetch the number of rows from the table
+            sample_query = f"SELECT COUNT(*) AS ROWS FROM {self.table_name} LIMIT 1"
             num_rows = pd.read_sql_query(sample_query, engine)
             logger.info(f"Number of rows inserted: {num_rows.values[0][0]}")
             engine.dispose()