diff --git a/job-bot.py b/job-bot.py index 6790a22..256a08d 100644 --- a/job-bot.py +++ b/job-bot.py @@ -10,7 +10,6 @@ import json import openai import graypy import logging -from airflow.operators.python import PythonVirtualenvOperator DEBUG = True @@ -32,17 +31,11 @@ graylog_handler = graypy.GELFUDPHandler('graylog.localdomain', 12201) graylog_handler.include_logger_name = True logger.addHandler(graylog_handler) -requirements = [ - "pymongo==4.6.1", - "graypy==2.1.0", - "openai==1.12.0" -] - with DAG( 'job_bot_api_dag', default_args=default_args, description='A DAG to fetch data from job-bot API and process it', - schedule=timedelta(hours=1), + schedule='*/10 * * * *', # Every 10 minutes start_date=datetime.now() - timedelta(days=1), # Changed to today-1 day catchup=False, tags=['job-bot', 'api'], @@ -82,7 +75,7 @@ with DAG( def insert_into_mongodb(api_results): mongo_username = "root" mongo_password = "passwd" - client = MongoClient(f"mongodb://{mongo_username}:{mongo_password}@localhost:27017/") + client = MongoClient(f"mongodb://{mongo_username}:{mongo_password}@job-bot.localdomain:27017/") db = client["job_bot_db"] collection = db["job-bot"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a6436e6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,141 @@ +a2wsgi==1.10.8 +aiohappyeyeballs==2.6.1 +aiohttp==3.11.18 +aiologic==0.14.0 +aiosignal==1.3.2 +aiosqlite==0.21.0 +alembic==1.15.2 +annotated-types==0.7.0 +anyio==4.9.0 +apache-airflow==3.0.1 +apache-airflow-core==3.0.1 +apache-airflow-providers-common-compat==1.7.0 +apache-airflow-providers-common-io==1.6.0 +apache-airflow-providers-common-sql==1.27.1 +apache-airflow-providers-smtp==2.1.0 +apache-airflow-providers-standard==1.2.0 +apache-airflow-task-sdk==1.0.1 +argcomplete==3.6.2 +asgiref==3.8.1 +asyncio==3.4.3 +attrs==25.3.0 +beautifulsoup4==4.13.4 +blinker==1.9.0 +cadwyn==5.3.3 +certifi==2025.4.26 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.1.8 +colorlog==6.9.0 +cron-descriptor==1.4.5 +croniter==6.0.0 +cryptography==45.0.2 +Deprecated==1.2.18 +dill==0.4.0 +distro==1.9.0 +dnspython==2.7.0 +email_validator==2.2.0 +fastapi==0.115.12 +fastapi-cli==0.0.7 +Flask==3.1.1 +frozenlist==1.6.0 +fsspec==2025.3.2 +googleapis-common-protos==1.70.0 +grapy==0.1.11 +greenlet==3.2.2 +grpcio==1.71.0 +gunicorn==23.0.0 +h11==0.16.0 +httpcore==1.0.9 +httptools==0.6.4 +httpx==0.28.1 +idna==3.10 +importlib_metadata==8.6.1 +itsdangerous==2.2.0 +Jinja2==3.1.6 +jiter==0.10.0 +jsonschema==4.23.0 +jsonschema-specifications==2025.4.1 +lazy-object-proxy==1.11.0 +libcst==1.7.0 +linkify-it-py==2.0.3 +lockfile==0.12.2 +Mako==1.3.10 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +methodtools==0.4.7 +more-itertools==10.7.0 +msgspec==0.19.0 +multidict==6.4.4 +openai==1.79.0 +opentelemetry-api==1.33.1 +opentelemetry-exporter-otlp==1.33.1 +opentelemetry-exporter-otlp-proto-common==1.33.1 +opentelemetry-exporter-otlp-proto-grpc==1.33.1 +opentelemetry-exporter-otlp-proto-http==1.33.1 +opentelemetry-proto==1.33.1 +opentelemetry-sdk==1.33.1 +opentelemetry-semantic-conventions==0.54b1 +packaging==25.0 +pathspec==0.12.1 +pendulum==3.1.0 +pluggy==1.6.0 +propcache==0.3.1 +protobuf==5.29.4 +psutil==7.0.0 +pycparser==2.22 +pydantic==2.11.4 +pydantic_core==2.33.2 +Pygments==2.19.1 +PyJWT==2.10.1 +pymongo==4.13.0 +python-daemon==3.1.2 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.0 +python-multipart==0.0.20 +python-slugify==8.0.4 +pytz==2025.2 +PyYAML==6.0.2 +referencing==0.36.2 +requests==2.32.3 +retryhttp==1.3.3 +rich==14.0.0 +rich-argparse==1.7.0 +rich-toolkit==0.14.6 +rpds-py==0.25.0 +setproctitle==1.3.6 +shellingham==1.5.4 +six==1.17.0 +sniffio==1.3.1 +soupsieve==2.7 +SQLAlchemy==1.4.54 +SQLAlchemy-JSONField==1.0.2 +SQLAlchemy-Utils==0.41.2 +sqlparse==0.5.3 +starlette==0.46.2 +structlog==25.3.0 +svcs==25.1.0 +tabulate==0.9.0 +tenacity==9.1.2 +termcolor==3.1.0 +text-unidecode==1.3 +tqdm==4.67.1 +typer==0.15.4 +types-requests==2.32.0.20250515 +typing-inspection==0.4.0 +typing_extensions==4.13.2 +tzdata==2025.2 +uc-micro-py==1.0.3 +universal_pathlib==0.2.6 +urllib3==2.4.0 +uuid6==2024.7.10 +uvicorn==0.34.2 +uvloop==0.21.0 +watchfiles==1.0.5 +websockets==15.0.1 +Werkzeug==3.1.3 +wirerope==1.0.0 +wrapt==1.17.2 +yarl==1.20.0 +zipp==3.21.0