- install Miniconda
- install g++
- create python3 enviroment
- install scrapy and scrapy-splash and faker and maybe selenium
- install docker and run splash
sudo apt install g++
conda create -n py3 python
conda activate py3
pip install Scrapy
pip install scrapy-splash
pip install faker
pip install selenium
sudo apt install docker.io
tmux
sudo docker run -p 8050:8050 scrapinghub/splash
- clone this repo
- change directory to this repo root
- git branch switch to
splash
- ensure your docker image running and python enviorment is alright
- start your crawl task(recommend using tmux)
git clone [email protected]:hund030/Acrawler.git
cd Acrawler
git checkout splash
tmux
scrapy runspider douban/spiders/replies_spider.py -o reply.json -s JOBDIR=~/logs