running well

job-bot.py

@@ -10,7 +10,6 @@ import json
 import openai
 import graypy
 import logging
-from airflow.operators.python import PythonVirtualenvOperator
 
 DEBUG = True
 
@@ -32,17 +31,11 @@ graylog_handler = graypy.GELFUDPHandler('graylog.localdomain', 12201)
 graylog_handler.include_logger_name = True
 logger.addHandler(graylog_handler)
 
-requirements = [
-    "pymongo==4.6.1",
-    "graypy==2.1.0",
-    "openai==1.12.0"
-]
-
 with DAG(
     'job_bot_api_dag',
     default_args=default_args,
     description='A DAG to fetch data from job-bot API and process it',
-    schedule=timedelta(hours=1),
+    schedule='*/10 * * * *',  # Every 10 minutes
     start_date=datetime.now() - timedelta(days=1),  # Changed to today-1 day
     catchup=False,
     tags=['job-bot', 'api'],
@@ -82,7 +75,7 @@ with DAG(
     def insert_into_mongodb(api_results):
         mongo_username = "root"
         mongo_password = "passwd"
-        client = MongoClient(f"mongodb://{mongo_username}:{mongo_password}@localhost:27017/")
+        client = MongoClient(f"mongodb://{mongo_username}:{mongo_password}@job-bot.localdomain:27017/")
         db = client["job_bot_db"]
         collection = db["job-bot"]
 

requirements.txt

@@ -0,0 +1,141 @@
+a2wsgi==1.10.8
+aiohappyeyeballs==2.6.1
+aiohttp==3.11.18
+aiologic==0.14.0
+aiosignal==1.3.2
+aiosqlite==0.21.0
+alembic==1.15.2
+annotated-types==0.7.0
+anyio==4.9.0
+apache-airflow==3.0.1
+apache-airflow-core==3.0.1
+apache-airflow-providers-common-compat==1.7.0
+apache-airflow-providers-common-io==1.6.0
+apache-airflow-providers-common-sql==1.27.1
+apache-airflow-providers-smtp==2.1.0
+apache-airflow-providers-standard==1.2.0
+apache-airflow-task-sdk==1.0.1
+argcomplete==3.6.2
+asgiref==3.8.1
+asyncio==3.4.3
+attrs==25.3.0
+beautifulsoup4==4.13.4
+blinker==1.9.0
+cadwyn==5.3.3
+certifi==2025.4.26
+cffi==1.17.1
+charset-normalizer==3.4.2
+click==8.1.8
+colorlog==6.9.0
+cron-descriptor==1.4.5
+croniter==6.0.0
+cryptography==45.0.2
+Deprecated==1.2.18
+dill==0.4.0
+distro==1.9.0
+dnspython==2.7.0
+email_validator==2.2.0
+fastapi==0.115.12
+fastapi-cli==0.0.7
+Flask==3.1.1
+frozenlist==1.6.0
+fsspec==2025.3.2
+googleapis-common-protos==1.70.0
+grapy==0.1.11
+greenlet==3.2.2
+grpcio==1.71.0
+gunicorn==23.0.0
+h11==0.16.0
+httpcore==1.0.9
+httptools==0.6.4
+httpx==0.28.1
+idna==3.10
+importlib_metadata==8.6.1
+itsdangerous==2.2.0
+Jinja2==3.1.6
+jiter==0.10.0
+jsonschema==4.23.0
+jsonschema-specifications==2025.4.1
+lazy-object-proxy==1.11.0
+libcst==1.7.0
+linkify-it-py==2.0.3
+lockfile==0.12.2
+Mako==1.3.10
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+mdurl==0.1.2
+methodtools==0.4.7
+more-itertools==10.7.0
+msgspec==0.19.0
+multidict==6.4.4
+openai==1.79.0
+opentelemetry-api==1.33.1
+opentelemetry-exporter-otlp==1.33.1
+opentelemetry-exporter-otlp-proto-common==1.33.1
+opentelemetry-exporter-otlp-proto-grpc==1.33.1
+opentelemetry-exporter-otlp-proto-http==1.33.1
+opentelemetry-proto==1.33.1
+opentelemetry-sdk==1.33.1
+opentelemetry-semantic-conventions==0.54b1
+packaging==25.0
+pathspec==0.12.1
+pendulum==3.1.0
+pluggy==1.6.0
+propcache==0.3.1
+protobuf==5.29.4
+psutil==7.0.0
+pycparser==2.22
+pydantic==2.11.4
+pydantic_core==2.33.2
+Pygments==2.19.1
+PyJWT==2.10.1
+pymongo==4.13.0
+python-daemon==3.1.2
+python-dateutil==2.9.0.post0
+python-dotenv==1.1.0
+python-multipart==0.0.20
+python-slugify==8.0.4
+pytz==2025.2
+PyYAML==6.0.2
+referencing==0.36.2
+requests==2.32.3
+retryhttp==1.3.3
+rich==14.0.0
+rich-argparse==1.7.0
+rich-toolkit==0.14.6
+rpds-py==0.25.0
+setproctitle==1.3.6
+shellingham==1.5.4
+six==1.17.0
+sniffio==1.3.1
+soupsieve==2.7
+SQLAlchemy==1.4.54
+SQLAlchemy-JSONField==1.0.2
+SQLAlchemy-Utils==0.41.2
+sqlparse==0.5.3
+starlette==0.46.2
+structlog==25.3.0
+svcs==25.1.0
+tabulate==0.9.0
+tenacity==9.1.2
+termcolor==3.1.0
+text-unidecode==1.3
+tqdm==4.67.1
+typer==0.15.4
+types-requests==2.32.0.20250515
+typing-inspection==0.4.0
+typing_extensions==4.13.2
+tzdata==2025.2
+uc-micro-py==1.0.3
+universal_pathlib==0.2.6
+urllib3==2.4.0
+uuid6==2024.7.10
+uvicorn==0.34.2
+uvloop==0.21.0
+watchfiles==1.0.5
+websockets==15.0.1
+Werkzeug==3.1.3
+wirerope==1.0.0
+wrapt==1.17.2
+yarl==1.20.0
+zipp==3.21.0