diff --git a/client/src/components/SingleJob.js b/client/src/components/SingleJob.js
index 3829515..49a3d14 100644
--- a/client/src/components/SingleJob.js
+++ b/client/src/components/SingleJob.js
@@ -22,20 +22,6 @@ class SingleJob extends Component {
.then(res => this.updateStatus());
};
- downloadFunc = () => {
- fetchWithUserToken(`/api/jobs/${this.props.id}/files`, {method:"GET"})
- .then(response => response.blob())
- .then(blob => {
- let url = window.URL.createObjectURL(blob);
- let a = document.createElement("a");
- a.href = url;
- a.download = this.props.id + ".zip";
- document.body.appendChild(a); // we need to append the element to the dom -> otherwise it will not work in firefox
- a.click();
- a.remove(); //afterwards we remove the element again
- });
- };
-
updateStatus = () => {
fetchWithUserToken(`/api/jobs/${this.props.id}`, {method:"GET"})
.then(res => {
@@ -89,7 +75,8 @@ class SingleJob extends Component {
+ href={`/api/jobs/${this.props.id}/download`}
+ >
Download
,
diff --git a/environment.yml b/environment.yml
new file mode 100644
index 0000000..596bbc4
--- /dev/null
+++ b/environment.yml
@@ -0,0 +1,180 @@
+name: wc-server
+channels:
+ - defaults
+dependencies:
+ - bzip2=1.0.8=h1de35cc_0
+ - ca-certificates=2021.10.26=hecd8cb5_2
+ - libcxx=12.0.0=h2f01273_0
+ - libffi=3.3=hb1e8313_2
+ - ncurses=6.3=hca72f7f_2
+ - openssl=1.1.1m=hca72f7f_0
+ - pip=21.2.4=py310hecd8cb5_0
+ - python=3.10.0=hdfd78df_3
+ - readline=8.1.2=hca72f7f_1
+ - setuptools=58.0.4=py310hecd8cb5_0
+ - sqlite=3.37.0=h707629a_0
+ - tk=8.6.11=h7bc2e8c_0
+ - tzdata=2021e=hda174b7_0
+ - xz=5.2.5=h1de35cc_0
+ - zlib=1.2.11=h4dc903c_4
+ - pip:
+ - aniso8601==9.0.1
+ - anyio==3.5.0
+ - appnope==0.1.2
+ - argcomplete==1.10.0
+ - argon2-cffi==21.3.0
+ - argon2-cffi-bindings==21.2.0
+ - asttokens==2.0.5
+ - attrs==20.3.0
+ - authlib==0.15.5
+ - automat==20.2.0
+ - babel==2.9.1
+ - backcall==0.2.0
+ - beautifulsoup4==4.8.0
+ - billiard==3.6.3.0
+ - black==22.1.0
+ - bleach==4.1.0
+ - cachetools==5.0.0
+ - certifi==2020.12.5
+ - cffi==1.14.4
+ - chardet==3.0.4
+ - charset-normalizer==2.0.11
+ - click==8.0.3
+ - constantly==15.1.0
+ - cryptography==3.3.2
+ - cssselect==1.1.0
+ - debugpy==1.5.1
+ - decorator==5.1.1
+ - defusedxml==0.7.1
+ - deprecated==1.2.13
+ - docx2txt==0.8
+ - ebooklib==0.17.1
+ - entrypoints==0.4
+ - executing==0.8.2
+ - extract-msg==0.23.1
+ - filelock==3.0.12
+ - flask==1.1.2
+ - flask-restful==0.3.8
+ - google==3.0.0
+ - google-api-core==2.4.0
+ - google-api-python-client==2.33.0
+ - google-auth==2.6.0
+ - google-auth-httplib2==0.1.0
+ - googleapis-common-protos==1.54.0
+ - h2==3.2.0
+ - hpack==3.0.0
+ - html5lib==1.1
+ - httplib2==0.20.2
+ - hyperframe==5.2.0
+ - hyperlink==20.0.1
+ - idna==2.10
+ - imapclient==2.1.0
+ - incremental==17.5.0
+ - ipykernel==6.8.0
+ - ipython==8.0.1
+ - ipython-genutils==0.2.0
+ - itemadapter==0.2.0
+ - itemloaders==1.0.4
+ - itsdangerous==2.0.1
+ - jedi==0.18.1
+ - jinja2==3.0.3
+ - jmespath==0.10.0
+ - json5==0.9.6
+ - jsonschema==4.4.0
+ - jupyter-client==7.1.2
+ - jupyter-core==4.9.1
+ - jupyter-server==1.13.4
+ - jupyterlab==3.2.9
+ - jupyterlab-pygments==0.1.2
+ - jupyterlab-server==2.10.3
+ - lxml==4.6.5
+ - markupsafe==2.0.1
+ - matplotlib-inline==0.1.3
+ - mistune==0.8.4
+ - mypy-extensions==0.4.3
+ - nbclassic==0.3.5
+ - nbclient==0.5.10
+ - nbconvert==6.4.1
+ - nbformat==5.1.3
+ - nest-asyncio==1.5.4
+ - notebook==6.4.8
+ - numpy==1.22.2
+ - olefile==0.46
+ - packaging==21.3
+ - pandas==1.4.0
+ - pandocfilters==1.5.0
+ - parsel==1.6.0
+ - parso==0.8.3
+ - pathspec==0.9.0
+ - pdfminer-six==20181108
+ - pexpect==4.8.0
+ - pickleshare==0.7.5
+ - pillow==9.0.0
+ - platformdirs==2.4.1
+ - priority==1.3.0
+ - prometheus-client==0.13.1
+ - prompt-toolkit==3.0.26
+ - protego==0.1.16
+ - protobuf==3.19.4
+ - ptyprocess==0.7.0
+ - pure-eval==0.2.2
+ - pyasn1==0.4.8
+ - pyasn1-modules==0.2.8
+ - pycparser==2.20
+ - pycryptodome==3.9.9
+ - pydispatcher==2.0.5
+ - pygments==2.11.2
+ - pyhamcrest==2.0.2
+ - pymongo==3.11.2
+ - pyopenssl==20.0.0
+ - pyparsing==3.0.7
+ - pyrsistent==0.18.1
+ - python-dateutil==2.8.2
+ - python-dotenv==0.19.2
+ - python-pptx==0.6.18
+ - pytz==2020.4
+ - pyzmq==22.3.0
+ - queuelib==1.5.0
+ - redis==4.1.2
+ - regex==2020.11.13
+ - requests==2.25.0
+ - requests-file==1.5.1
+ - rq==1.8.1
+ - rsa==4.8
+ - scrapy==2.5.1
+ - scrapyscript==1.1.0
+ - send2trash==1.8.0
+ - service-identity==18.1.0
+ - six==1.12.0
+ - sniffio==1.2.0
+ - sortedcontainers==2.3.0
+ - soupsieve==2.0.1
+ - speechrecognition==3.8.1
+ - stack-data==0.1.4
+ - terminado==0.13.1
+ - testpath==0.5.0
+ - textract==1.6.3
+ - tldextract==3.1.0
+ - tomli==2.0.0
+ - tornado==6.1
+ - traitlets==5.1.1
+ - twisted==20.3.0
+ - tzlocal==1.5.1
+ - uritemplate==4.1.1
+ - urllib3==1.26.5
+ - w3lib==1.22.0
+ - wcwidth==0.2.5
+ - webencodings==0.5.1
+ - websocket-client==1.2.3
+ - werkzeug==2.0.2
+ - wheel==0.37.1
+ - wrapt==1.13.3
+ - xlrd==1.2.0
+ - xlsxwriter==1.3.7
+ - zope-interface==5.2.0
+variables:
+ CLIENT_ORIGIN: http://localhost:3000
+ MONGO_URI: mongodb://localhost:27017
+ REACT_APP_SERVER_URL: http://localhost:5000
+ SERVER_PORT: '5000'
+prefix: /Users/miclin/opt/anaconda3/envs/wc-server
diff --git a/server/__init__.py b/server/__init__.py
index ef340e8..415c2d6 100644
--- a/server/__init__.py
+++ b/server/__init__.py
@@ -18,12 +18,14 @@ def create_app(test_config=None):
app.register_blueprint(job_interfaces.bp)
from .jobs import actions as job_actions
app.register_blueprint(job_actions.bp)
+ from .jobs import download as download
+ app.register_blueprint(download.bp)
return app
def validate_server_settings():
- assert (GOOGLE_OAUTH_CLIENT_URL is not None, "Missing Google oauth client ID")
+ assert (GOOGLE_OAUTH_CLIENT_ID is not None, "Missing Google oauth client ID")
assert (FLASK_ENV != "production" or not DEBUG_NO_AUTH_ENABLED,
"No auth mode mustn't be enabled for production environment")
if DEBUG_NO_AUTH_ENABLED:
diff --git a/server/crawler/getzip.py b/server/crawler/getzip.py
deleted file mode 100755
index 44a573e..0000000
--- a/server/crawler/getzip.py
+++ /dev/null
@@ -1,41 +0,0 @@
-import pymongo
-import gridfs
-from server import settings
-from tempfile import TemporaryDirectory
-from os.path import join
-import shutil
-import time
-
-def gridfs_bucket(db,foldername, bucket, rq_id):
- fs = gridfs.GridFSBucket(db,bucket)
- for f in fs.find({"rq_id":rq_id}):
- with open(join(foldername,f.filename),"wb") as localfile:
- fs.download_to_stream(f._id, localfile)
-
-def url_to_filename(s):
- s = s.replace("/","|")
- return "".join( x for x in s if (x.isalnum() or x in "|._- "))
-
-def text_collection(db,foldername, collection, rq_id):
- records = db[collection]
- for f in records.find({"rq_id":rq_id}):
- filename = url_to_filename(f["url"])+".txt"
- with open(join(foldername,filename),"w") as localfile:
- localfile.write(f["text"])
-
-def getzip(rq_id):
- connection = pymongo.MongoClient(
- settings.MONGO_URI,
- username = settings.MONGO_USERNAME,
- password = settings.MONGO_PASSWORD
- )
- db = connection[settings.MONGODB_DB]
- print("CONNECTED TO MONGO DB")
- with TemporaryDirectory() as dir:
- gridfs_bucket(db, dir,settings.MONGODB_COLLECTION_IMAGES,rq_id)
- gridfs_bucket(db, dir, settings.MONGODB_COLLECTION_FILES, rq_id)
- text_collection(db, dir, settings.MONGODB_COLLECTION_TEXT, rq_id)
- zip_path = join("crawled_data_","files",rq_id+'_'+str(time.time()))
- shutil.make_archive(zip_path, 'zip', dir)
-
- return zip_path + ".zip"
diff --git a/server/crawler/items.py b/server/crawler/items.py
index 6d9a3f1..3dea2aa 100755
--- a/server/crawler/items.py
+++ b/server/crawler/items.py
@@ -5,6 +5,7 @@
from scrapy.item import Item, Field
+
class CrawlerItem(Item):
"""
Represents the object parsed by the spider.
diff --git a/server/crawler/output.py b/server/crawler/output.py
new file mode 100644
index 0000000..2a28124
--- /dev/null
+++ b/server/crawler/output.py
@@ -0,0 +1,52 @@
+import shutil
+
+import pymongo
+import gridfs
+import os
+import csv
+import server.settings as settings
+from shutil import make_archive
+from tempfile import TemporaryDirectory
+from bson import ObjectId
+
+db = pymongo.MongoClient(
+ settings.MONGO_URI, username=settings.MONGO_USERNAME,
+ password=settings.MONGO_PASSWORD
+)[settings.MONGO_DB]
+
+
+def bucket_export(folder, bucket_name, job_id):
+ bucket_folder_path = os.path.join(folder, f"{bucket_name}/")
+ os.makedirs(os.path.dirname(bucket_folder_path), exist_ok=True)
+ fs = gridfs.GridFSBucket(db, bucket_name)
+ for f in fs.find({"job_id": job_id}):
+ with open(os.path.join(bucket_folder_path, f.filename), "wb") \
+ as local_file:
+ fs.download_to_stream(f._id, local_file)
+
+
+def items_export(folder, job_id):
+ docs = db[settings.MONGO_COLLECTION_ITEMS]\
+ .find({"job_id": job_id})
+
+ if docs.count() == 0:
+ return
+
+ fields = list(docs[0].keys())
+ output_file = os.path.join(folder, f"{job_id}.csv")
+ with open(output_file, "w", newline='') as csvfile:
+ writer = csv.DictWriter(csvfile, fieldnames=fields, extrasaction="ignore")
+ writer.writeheader()
+ writer.writerows(docs)
+
+
+def get_zip(job_id: str) -> str:
+ dir_path = os.path.dirname(os.path.realpath(__file__))
+ result_file_path = os.path.join(dir_path, "results", job_id)
+ with TemporaryDirectory() as temp_dir:
+ bucket_export(temp_dir, settings.MONGO_BUCKET_IMAGES, job_id)
+ bucket_export(temp_dir, settings.MONGO_BUCKET_FILES, job_id)
+ items_export(temp_dir, job_id)
+ shutil.make_archive(result_file_path, "zip", temp_dir)
+
+ return f"{result_file_path}.zip"
diff --git a/server/crawler/pipelines.py b/server/crawler/pipelines.py
index 8bd9be7..088b99e 100755
--- a/server/crawler/pipelines.py
+++ b/server/crawler/pipelines.py
@@ -32,14 +32,10 @@
class MongoDBPipeline:
- COLLECTION = "items"
- BUCKET_COLLECTION = "bucketItems"
- OTHER_COLLECTION = "otherItems"
def __init__(self, mongo_uri, mongo_db, mongo_user='admin', mongo_pwd='', mongo_repl=False, mongo_repl_name=''):
self.client = None
self.db = None
- self.grid_fs = None
self.mongo_uri = mongo_uri
self.mongo_db = mongo_db
self.mongo_user = mongo_user
@@ -71,8 +67,6 @@ def open_spider(self, spider):
self.client = pymongo.MongoClient(self.mongo_uri, username=self.mongo_user,
password=self.mongo_password)
self.db = self.client[self.mongo_db]
- self.grid_fs = gridfs.GridFS(self.db,
- collection=self.BUCKET_COLLECTION)
print("Connected")
def close_spider(self, spider):
@@ -98,23 +92,24 @@ def process_item(self, item, spider):
query = {'url': item['url']}
if not isinstance(item, CrawlerItem):
- print("Not an instance of CrawlerItem")
- print(item['url'])
- self.db[self.OTHER_COLLECTION].replace_one(query, adapted_item, upsert=True)
+ logging.debug("Not an instance of CrawlerItem")
+ logging.debug(item['url'])
+ self.db[settings.MONGO_COLLECTION_OTHERS]\
+ .replace_one(query, adapted_item, upsert=True)
return item
# upsert=True means insert the document if the query doesn't find a match.
- self.db[self.COLLECTION].replace_one(
+ self.db[settings.MONGO_COLLECTION_ITEMS].replace_one(
query, adapted_item, upsert=True
)
urls = item["image_urls"]
if type(urls) is list and len(urls) != 0:
- self.save_to_bucket(urls, "images", spider)
+ self.save_to_bucket(urls, settings.MONGO_BUCKET_IMAGES, spider)
urls = item["file_urls"]
if type(urls) is list and len(urls) != 0:
- self.save_to_bucket(urls, "files", spider)
+ self.save_to_bucket(urls, settings.MONGO_BUCKET_FILES, spider)
logging.debug(f"MongoDB: Inserted {item['url']}.")
return item
@@ -123,7 +118,8 @@ def save_to_bucket(self, urls, bucket_name, spider):
for url in urls:
mime_type = mimetypes.guess_type(url)[0]
request = requests.get(url, stream=True)
- self.grid_fs.put(request.raw, contentType=mime_type,
- user=spider.user if hasattr(spider, "user") else None,
- job_id=spider.job_id if hasattr(spider, "job_id") else None,
- filename=os.path.basename(url), bucket_name=bucket_name)
+ fs = gridfs.GridFS(self.db, bucket_name)
+ fs.put(request.raw, contentType=mime_type,
+ user=spider.user if hasattr(spider, "user") else None,
+ job_id=spider.job_id if hasattr(spider, "job_id") else None,
+ filename=os.path.basename(url), bucket_name=bucket_name)
diff --git a/server/crawler/results/.gitignore b/server/crawler/results/.gitignore
new file mode 100644
index 0000000..86d0cb2
--- /dev/null
+++ b/server/crawler/results/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file
diff --git a/server/crawler/tracking.py b/server/crawler/tracking.py
index 3ffb8ab..2bf9bce 100644
--- a/server/crawler/tracking.py
+++ b/server/crawler/tracking.py
@@ -148,8 +148,8 @@ def get_all_jobs(self, user: str) -> {str, CrawlJob}:
mongo_uri=settings.MONGO_URI,
mongo_user=settings.MONGO_USERNAME,
mongo_pass=settings.MONGO_PASSWORD,
- db_name=settings.MONGODB_DB,
- jobs_collection=settings.MONGODB_COLLECTION_JOBS
+ db_name=settings.MONGO_DB,
+ jobs_collection=settings.MONGO_COLLECTION_JOBS
)
if __name__ == '__main__':
diff --git a/server/jobs/actions.py b/server/jobs/actions.py
index 7da887a..3795132 100644
--- a/server/jobs/actions.py
+++ b/server/jobs/actions.py
@@ -1,4 +1,3 @@
-import json
from functools import wraps
from flask import Blueprint, request, jsonify
from server.crawler.tracking import job_repository, CrawlJobStatus
@@ -77,8 +76,3 @@ def job_cancel(job_id):
message=f"{job_id} is not an active job: status={status.value}"
), 405
-
-@bp.route("//result", methods=["GET"])
-def job_result(job_id):
- status = job_repository.get_status(job_id)
- # TODO: Serialize and send results
diff --git a/server/jobs/download.py b/server/jobs/download.py
new file mode 100644
index 0000000..58ea28b
--- /dev/null
+++ b/server/jobs/download.py
@@ -0,0 +1,39 @@
+import os.path
+
+from server.crawler.output import get_zip
+from .utils import token_required
+from server.crawler.tracking import job_repository
+from server.jobs.actions import validate_job_id
+
+from flask import Blueprint, send_file
+
+bp = Blueprint("job_download", __name__, url_prefix="/api/jobs")
+
+
+@bp.before_request
+@token_required
+@validate_job_id
+def before_request():
+ """Validate user and job_id at all action endpoints."""
+ pass
+
+
+@bp.after_request
+def after_request(response):
+ header = response.headers
+ header["Access-Control-Allow-Credentials"] = True
+ header["Access-Control-Allow-Headers"] = "*"
+ header["Access-Control-Allow-Methods"] = "*"
+ return response
+
+
+@bp.route("//download", methods=["GET"])
+def download_job_result(job_id):
+ status = job_repository.get_status(job_id)
+ zip_path = get_zip(job_id)
+ response = send_file(zip_path, as_attachment=True)
+ response.headers["Content-Type"] = "application/zip"
+ filename = os.path.basename(zip_path)
+ response.headers["Content-Disposition"] = \
+ f"attachment; filename=\"{filename}\""
+ return response
diff --git a/server/jobs/utils.py b/server/jobs/utils.py
index e88f229..7b96f80 100644
--- a/server/jobs/utils.py
+++ b/server/jobs/utils.py
@@ -23,7 +23,7 @@ def decorator(*args, **kwargs):
try:
id_info = id_token.verify_oauth2_token(
- token, requests.Request(), GOOGLE_OAUTH_CLIENT_URL
+ token, requests.Request(), GOOGLE_OAUTH_CLIENT_ID
)
g.user = id_info["email"]
return f(*args, **kwargs)
diff --git a/server/settings.py b/server/settings.py
index 4854409..97258de 100755
--- a/server/settings.py
+++ b/server/settings.py
@@ -82,11 +82,14 @@
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
-MONGODB_DB = 'crawlerSpider'
-MONGODB_COLLECTION_IMAGES = "images"
-MONGODB_COLLECTION_FILES = "files"
-MONGODB_COLLECTION_TEXT = "text"
-MONGODB_COLLECTION_JOBS = "jobs"
+MONGO_DB = 'crawlerSpider'
+MONGO_BUCKET_IMAGES = "images"
+MONGO_BUCKET_FILES = "files"
+MONGO_COLLECTION_ITEMS = "items"
+# MONGO_COLLECTION_TEXT = "text"
+# Currently not used
+MONGO_COLLECTION_JOBS = "jobs"
+MONGO_COLLECTION_OTHERS = "others"
# running locally without containers
MONGO_URI = os.getenv('MONGO_URI') or 'mongodb://localhost:27017'
@@ -112,7 +115,7 @@
IMAGES_MIN_HEIGHT = 150
IMAGES_MIN_WIDTH = 150
-GOOGLE_OAUTH_CLIENT_URL = os.getenv('GOOGLE_OAUTH_CLIENT_ID') or None
+GOOGLE_OAUTH_CLIENT_ID = os.getenv('GOOGLE_OAUTH_CLIENT_ID') or None
FLASK_ENV = os.getenv("FLASK_ENV") or "production"
DEBUG_NO_AUTH_ENABLED = os.getenv("DEBUG_NO_AUTH_ENABLED") == "True"