I'm running Jupyterhub in a docker swarm, on an OpenStack base. I have 1 manager & 2 worker nodes.
I can happily create the 'Hub docker image, and log into it.... however that hub is not able to spawn notebooks.... and yes, the notebook has been pulled into the Swarm.
FROM jupyterhub/jupyterhub
RUN apt-get update
RUN apt-get -y install python3-pip
# Temporary add these usful tools while developing
RUN apt-get -y install vim less net-tools
RUN useradd -ms /bin/bash test
RUN echo test:password | chpasswd
RUN git clone https://github.com/cassinyio/SwarmSpawner.git
WORKDIR SwarmSpawner
RUN pip install -r requirements.txt
RUN pip install .
WORKDIR /srv/jupyterhub
COPY config/jupyterhub_config.py /srv/jupyterhub/jupyterhub_config.py
CMD jupyterhub --debug > log.out 2>&1
import subprocess
import os
import errno
import stat
c = get_config()
pwd = os.path.dirname(__file__)
c.JupyterHub.spawner_class = 'cassinyspawner.SwarmSpawner'
c.JupyterHub.ip = '0.0.0.0'
c.JupyterHub.hub_ip = '0.0.0.0'
c.JupyterHub.cleanup_servers = False
c.SwarmSpawner.start_timeout = 60 * 5
c.SwarmSpawner.jupyterhub_service_name = 'jupyterhub'
c.SwarmSpawner.networks = ["naas"]
notebook_dir = os.environ.get('NOTEBOOK_DIR') or '/home/test'
c.SwarmSpawner.notebook_dir = notebook_dir
mounts = [{'type' : 'volume',
'source' : 'jupyterhub-user-{username}',
'target' : notebook_dir}]
c.SwarmSpawner.container_spec = {
'args' : '/usr/local/bin/start-singleuser.sh',
'Image' : 'jupyterhub/singleuser',
'mounts' : mounts
}
c.SwarmSpawner.resource_spec = {
'cpu_limit' : 1000,
'mem_limit' : int(512 * 1e6),
'cpu_reservation' : 1000,
'mem_reservation' : int(512 * 1e6),
}
[D 2017-04-21 14:39:40.317 JupyterHub swarmspawner:221] Getting Docker service 'jupyter-098f6bcd4621d373cade4e832627b4f6-1'
[I 2017-04-21 14:39:40.321 JupyterHub swarmspawner:229] Docker service 'jupyter-098f6bcd4621d373cade4e832627b4f6-1' is gone
[W 2017-04-21 14:39:40.321 JupyterHub swarmspawner:194] Docker service not found
[E 2017-04-21 14:39:40.353 JupyterHub web:1548] Uncaught exception POST /hub/login?next= (10.255.0.2)
HTTPServerRequest(protocol='http', host='172.16.48.141:8000', method='POST', uri='/hub/login?next=', version='HTTP/1.1', remote_ip='10.255.0.2', headers={'Content-Type': 'application/x-www-form-urlencoded', 'X-Forwarded-Host': '172.16.48.141:8000', 'Accept-Language': 'en-US,en;q=0.5', 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0', 'Cookie': '_ga=GA1.1.103564032.1490776825; toggle=true', 'Upgrade-Insecure-Requests': '1', 'X-Forwarded-Proto': 'http', 'Content-Length': '31', 'Referer': 'http://172.16.48.141:8000/hub/login', 'X-Forwarded-Port': '8000', 'Connection': 'close', 'Accept-Encoding': 'gzip, deflate', 'Host': '172.16.48.141:8000', 'X-Forwarded-For': '10.255.0.2', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'})
Traceback (most recent call last):
File "/opt/conda/lib/python3.5/site-packages/tornado/web.py", line 1469, in _execute
result = yield result
File "/opt/conda/lib/python3.5/site-packages/jupyterhub/handlers/login.py", line 83, in post
yield self.spawn_single_user(user)
File "/opt/conda/lib/python3.5/site-packages/jupyterhub/handlers/base.py", line 325, in spawn_single_user
yield gen.with_timeout(timedelta(seconds=self.slow_spawn_timeout), f)
File "/opt/conda/lib/python3.5/site-packages/jupyterhub/user.py", line 310, in spawn
raise e
File "/opt/conda/lib/python3.5/site-packages/jupyterhub/user.py", line 278, in spawn
ip_port = yield gen.with_timeout(timedelta(seconds=spawner.start_timeout), f)
File "/opt/conda/lib/python3.5/site-packages/cassinyspawner/swarmspawner.py", line 311, in start
networks=networks)
File "/opt/conda/lib/python3.5/concurrent/futures/_base.py", line 398, in result
return self.__get_result()
File "/opt/conda/lib/python3.5/concurrent/futures/_base.py", line 357, in __get_result
raise self._exception
File "/opt/conda/lib/python3.5/concurrent/futures/thread.py", line 55, in run
result = self.fn(*self.args, **self.kwargs)
File "/opt/conda/lib/python3.5/site-packages/cassinyspawner/swarmspawner.py", line 180, in _docker
return m(*args, **kwargs)
File "/opt/conda/lib/python3.5/site-packages/docker/utils/decorators.py", line 35, in wrapper
return f(self, *args, **kwargs)
File "/opt/conda/lib/python3.5/site-packages/docker/api/service.py", line 84, in create_service
self._post_json(url, data=data, headers=headers), True
File "/opt/conda/lib/python3.5/site-packages/docker/api/client.py", line 220, in _result
self._raise_for_status(response)
File "/opt/conda/lib/python3.5/site-packages/docker/api/client.py", line 216, in _raise_for_status
raise create_api_error_from_http_exception(e)
File "/opt/conda/lib/python3.5/site-packages/docker/errors.py", line 30, in create_api_error_from_http_exception
raise cls(e, response=response, explanation=explanation)
docker.errors.APIError: 500 Server Error: Internal Server Error ("json: cannot unmarshal string into Go value of type []string")
resp = yield self.docker('create_service',
task_tmpl,
name=self.service_name,
networks=networks)
[I 2017-04-21 14:39:40.312 JupyterHub swarmspawner:307] Call Docker create_service task_template: {'ContainerSpec': {'Args': '/usr/local/bin/start-singleuser.sh',
'Command': None,
'Env': ['JPY_BASE_URL=/user/test',
'JPY_USER=test',
'JPY_COOKIE_NAME=jupyter-hub-token-test',
'JUPYTERHUB_CLIENT_ID=user-test',
'JPY_HUB_PREFIX=/hub/',
'JUPYTERHUB_API_TOKEN=e80c8e73010349bb85d846958b667bca',
'JUPYTERHUB_HOST=',
'NOTEBOOK_DIR=/home/test',
'JUPYTERHUB_OAUTH_CALLBACK_URL=/user/test/oauth_callback',
'JPY_API_TOKEN=e80c8e73010349bb85d846958b667bca',
'JPY_HUB_API_URL=http://jupyterhub:8081/hub/api'],
'Image': 'jupyterhub/singleuser',
'Mounts': [{'ReadOnly': False,
'Source': 'jupyterhub-user-098f6bcd4621d373cade4e832627b4f6',
'Target': '/home/test',
'Type': 'volume'}]},
'Resources': {'Limits': {'MemoryBytes': 512000000, 'NanoCPUs': 1000},
'Reservations': {'MemoryBytes': 512000000, 'NanoCPUs': 1000}}},
name:'jupyter-098f6bcd4621d373cade4e832627b4f6-1',
networks: ['naas']