running well
This commit is contained in:
Jason
parent
b4c2d0922a
commit
23b7c289bf
11
job-bot.py
11
job-bot.py
@ -10,7 +10,6 @@ import json
|
||||
import openai
|
||||
import graypy
|
||||
import logging
|
||||
from airflow.operators.python import PythonVirtualenvOperator
|
||||
|
||||
DEBUG = True
|
||||
|
||||
@ -32,17 +31,11 @@ graylog_handler = graypy.GELFUDPHandler('graylog.localdomain', 12201)
|
||||
graylog_handler.include_logger_name = True
|
||||
logger.addHandler(graylog_handler)
|
||||
|
||||
requirements = [
|
||||
"pymongo==4.6.1",
|
||||
"graypy==2.1.0",
|
||||
"openai==1.12.0"
|
||||
]
|
||||
|
||||
with DAG(
|
||||
'job_bot_api_dag',
|
||||
default_args=default_args,
|
||||
description='A DAG to fetch data from job-bot API and process it',
|
||||
schedule=timedelta(hours=1),
|
||||
schedule='*/10 * * * *', # Every 10 minutes
|
||||
start_date=datetime.now() - timedelta(days=1), # Changed to today-1 day
|
||||
catchup=False,
|
||||
tags=['job-bot', 'api'],
|
||||
@ -82,7 +75,7 @@ with DAG(
|
||||
def insert_into_mongodb(api_results):
|
||||
mongo_username = "root"
|
||||
mongo_password = "passwd"
|
||||
client = MongoClient(f"mongodb://{mongo_username}:{mongo_password}@localhost:27017/")
|
||||
client = MongoClient(f"mongodb://{mongo_username}:{mongo_password}@job-bot.localdomain:27017/")
|
||||
db = client["job_bot_db"]
|
||||
collection = db["job-bot"]
|
||||
|
||||
|
||||
141
requirements.txt
Normal file
141
requirements.txt
Normal file
@ -0,0 +1,141 @@
|
||||
a2wsgi==1.10.8
|
||||
aiohappyeyeballs==2.6.1
|
||||
aiohttp==3.11.18
|
||||
aiologic==0.14.0
|
||||
aiosignal==1.3.2
|
||||
aiosqlite==0.21.0
|
||||
alembic==1.15.2
|
||||
annotated-types==0.7.0
|
||||
anyio==4.9.0
|
||||
apache-airflow==3.0.1
|
||||
apache-airflow-core==3.0.1
|
||||
apache-airflow-providers-common-compat==1.7.0
|
||||
apache-airflow-providers-common-io==1.6.0
|
||||
apache-airflow-providers-common-sql==1.27.1
|
||||
apache-airflow-providers-smtp==2.1.0
|
||||
apache-airflow-providers-standard==1.2.0
|
||||
apache-airflow-task-sdk==1.0.1
|
||||
argcomplete==3.6.2
|
||||
asgiref==3.8.1
|
||||
asyncio==3.4.3
|
||||
attrs==25.3.0
|
||||
beautifulsoup4==4.13.4
|
||||
blinker==1.9.0
|
||||
cadwyn==5.3.3
|
||||
certifi==2025.4.26
|
||||
cffi==1.17.1
|
||||
charset-normalizer==3.4.2
|
||||
click==8.1.8
|
||||
colorlog==6.9.0
|
||||
cron-descriptor==1.4.5
|
||||
croniter==6.0.0
|
||||
cryptography==45.0.2
|
||||
Deprecated==1.2.18
|
||||
dill==0.4.0
|
||||
distro==1.9.0
|
||||
dnspython==2.7.0
|
||||
email_validator==2.2.0
|
||||
fastapi==0.115.12
|
||||
fastapi-cli==0.0.7
|
||||
Flask==3.1.1
|
||||
frozenlist==1.6.0
|
||||
fsspec==2025.3.2
|
||||
googleapis-common-protos==1.70.0
|
||||
grapy==0.1.11
|
||||
greenlet==3.2.2
|
||||
grpcio==1.71.0
|
||||
gunicorn==23.0.0
|
||||
h11==0.16.0
|
||||
httpcore==1.0.9
|
||||
httptools==0.6.4
|
||||
httpx==0.28.1
|
||||
idna==3.10
|
||||
importlib_metadata==8.6.1
|
||||
itsdangerous==2.2.0
|
||||
Jinja2==3.1.6
|
||||
jiter==0.10.0
|
||||
jsonschema==4.23.0
|
||||
jsonschema-specifications==2025.4.1
|
||||
lazy-object-proxy==1.11.0
|
||||
libcst==1.7.0
|
||||
linkify-it-py==2.0.3
|
||||
lockfile==0.12.2
|
||||
Mako==1.3.10
|
||||
markdown-it-py==3.0.0
|
||||
MarkupSafe==3.0.2
|
||||
mdurl==0.1.2
|
||||
methodtools==0.4.7
|
||||
more-itertools==10.7.0
|
||||
msgspec==0.19.0
|
||||
multidict==6.4.4
|
||||
openai==1.79.0
|
||||
opentelemetry-api==1.33.1
|
||||
opentelemetry-exporter-otlp==1.33.1
|
||||
opentelemetry-exporter-otlp-proto-common==1.33.1
|
||||
opentelemetry-exporter-otlp-proto-grpc==1.33.1
|
||||
opentelemetry-exporter-otlp-proto-http==1.33.1
|
||||
opentelemetry-proto==1.33.1
|
||||
opentelemetry-sdk==1.33.1
|
||||
opentelemetry-semantic-conventions==0.54b1
|
||||
packaging==25.0
|
||||
pathspec==0.12.1
|
||||
pendulum==3.1.0
|
||||
pluggy==1.6.0
|
||||
propcache==0.3.1
|
||||
protobuf==5.29.4
|
||||
psutil==7.0.0
|
||||
pycparser==2.22
|
||||
pydantic==2.11.4
|
||||
pydantic_core==2.33.2
|
||||
Pygments==2.19.1
|
||||
PyJWT==2.10.1
|
||||
pymongo==4.13.0
|
||||
python-daemon==3.1.2
|
||||
python-dateutil==2.9.0.post0
|
||||
python-dotenv==1.1.0
|
||||
python-multipart==0.0.20
|
||||
python-slugify==8.0.4
|
||||
pytz==2025.2
|
||||
PyYAML==6.0.2
|
||||
referencing==0.36.2
|
||||
requests==2.32.3
|
||||
retryhttp==1.3.3
|
||||
rich==14.0.0
|
||||
rich-argparse==1.7.0
|
||||
rich-toolkit==0.14.6
|
||||
rpds-py==0.25.0
|
||||
setproctitle==1.3.6
|
||||
shellingham==1.5.4
|
||||
six==1.17.0
|
||||
sniffio==1.3.1
|
||||
soupsieve==2.7
|
||||
SQLAlchemy==1.4.54
|
||||
SQLAlchemy-JSONField==1.0.2
|
||||
SQLAlchemy-Utils==0.41.2
|
||||
sqlparse==0.5.3
|
||||
starlette==0.46.2
|
||||
structlog==25.3.0
|
||||
svcs==25.1.0
|
||||
tabulate==0.9.0
|
||||
tenacity==9.1.2
|
||||
termcolor==3.1.0
|
||||
text-unidecode==1.3
|
||||
tqdm==4.67.1
|
||||
typer==0.15.4
|
||||
types-requests==2.32.0.20250515
|
||||
typing-inspection==0.4.0
|
||||
typing_extensions==4.13.2
|
||||
tzdata==2025.2
|
||||
uc-micro-py==1.0.3
|
||||
universal_pathlib==0.2.6
|
||||
urllib3==2.4.0
|
||||
uuid6==2024.7.10
|
||||
uvicorn==0.34.2
|
||||
uvloop==0.21.0
|
||||
watchfiles==1.0.5
|
||||
websockets==15.0.1
|
||||
Werkzeug==3.1.3
|
||||
wirerope==1.0.0
|
||||
wrapt==1.17.2
|
||||
yarl==1.20.0
|
||||
zipp==3.21.0
|
||||
Loading…
x
Reference in New Issue
Block a user