Merge pull request #21 from cuebook/RemoveEFS

Removed EFS dependency from zeppelin job server

Author: vikrantcue

Dockerfile

@@ -3,7 +3,7 @@ FROM node:12-alpine as builder
 WORKDIR /app
 ENV PATH /app/node_modules/.bin:$PATH
 COPY ui/package.json /app/package.json
-RUN npm install --silent
+RUN npm install
 COPY ui /app
 
 RUN npm run build

api/app/settings.py

@@ -156,7 +156,7 @@
 REDIS_BROKER_URL = os.environ.get("REDIS_BROKER_URL", "redis://redis:6379/0")
 
 # s3 file settings
-S3_BUCKET_NAME = os.environ.get("S3_BUCKET_NAME", None)
+S3_BUCKET_NAME = os.environ.get("S3_BUCKET_NAME", "")
 S3_FILES_PREFIX = os.environ.get("S3_FILES_PREFIX", "files/")
 
 # Postgres configuration for Schema Browser

api/seeddata/crontabschedule.json

@@ -6,8 +6,8 @@
       "minute": "0",
       "hour": "0",
       "day_of_week": "0",
-      "day_of_month": "30",
-      "month_of_year": "2",
+      "day_of_month": "1",
+      "month_of_year": "1",
       "timezone": "Asia/Kolkata"
     }
   },
@@ -18,4 +18,4 @@
       "name": "No Schedule"
     }
   }
-]
+]

api/utils/kubernetesTemplates/zeppelinServer.yaml

@@ -9,21 +9,26 @@ metadata:
 spec:
   serviceAccountName: zeppelin-server
   volumes:
-    - name: zeppelin-server-notebook-volume
-      persistentVolumeClaim:
-        claimName: cuelake-notebook-efs-pvc
-    - name: zeppelin-server-conf
-      persistentVolumeClaim:
-        claimName: cuelake-conf-efs-pvc
-    - name: nginx-conf
-      configMap:
-        name: zeppelin-server-conf
-        items:
-          - key: nginx.conf
-            path: nginx.conf
+    - name: shared-notebook
+      emptyDir:
+    - name: shared-conf
+      emptyDir:
+  initContainers:
+    - name: init
+      image: bitnami/kubectl
+      command: ['sh', '-c']
+      volumeMounts:
+        - name: shared-notebook
+          mountPath: /shared-notebook
+        - name: shared-conf
+          mountPath: /shared-conf
+      args:
+        - >-
+          kubectl cp $(kubectl get pods | grep zeppelin-server | awk '{print $1}' ):/zeppelin/conf /shared-conf &&
+          kubectl cp $(kubectl get pods | grep zeppelin-server | awk '{print $1}' ):/zeppelin/notebook /shared-notebook
   containers:
     - name: zeppelin-server
-      image: 'cuebook/zeppelin-server:0.9.0'
+      image: 'cuebook/zeppelin-server-lite:0.2'
       command:
         - sh
         - '-c'
@@ -58,9 +63,9 @@ spec:
               apiVersion: v1
               fieldPath: metadata.name
       volumeMounts:
-        - name: zeppelin-server-notebook-volume
+        - name: shared-notebook
           mountPath: /zeppelin/notebook
-        - name: zeppelin-server-conf
+        - name: shared-conf
           mountPath: /zeppelin/conf
       lifecycle:
         preStop:
@@ -78,46 +83,3 @@ spec:
         requests:
           memory: "1024Mi"
           cpu: "250m"
-    - name: zeppelin-server-gateway
-      image: 'nginx:1.14.0'
-      command:
-        - /bin/sh
-        - '-c'
-      args:
-        - >-
-          cp -f /tmp/conf/nginx.conf /etc/nginx/nginx.conf; sed -i -e
-          "s/SERVICE_DOMAIN/$SERVICE_DOMAIN/g" /etc/nginx/nginx.conf; sed -i -e
-          "s/NAMESPACE/$(cat
-          /var/run/secrets/kubernetes.io/serviceaccount/namespace)/g"
-          /etc/nginx/nginx.conf; cat /etc/nginx/nginx.conf; /usr/sbin/nginx
-      env:
-        - name: SERVICE_DOMAIN
-          valueFrom:
-            configMapKeyRef:
-              name: zeppelin-server-conf-map
-              key: SERVICE_DOMAIN
-      volumeMounts:
-        - name: nginx-conf
-          mountPath: /tmp/conf
-      lifecycle:
-        preStop:
-          exec:
-            command:
-              - /usr/sbin/nginx
-              - '-s'
-              - quit
-      terminationMessagePath: /dev/termination-log
-      terminationMessagePolicy: File
-      imagePullPolicy: IfNotPresent
-    - name: dnsmasq
-      image: 'janeczku/go-dnsmasq:release-1.0.5'
-      args:
-        - '--listen'
-        - '127.0.0.1:53'
-        - '--default-resolver'
-        - '--append-search-domains'
-        - '--hostsfile=/etc/hosts'
-        - '--verbose'
-      terminationMessagePath: /dev/termination-log
-      terminationMessagePolicy: File
-      imagePullPolicy: IfNotPresent

api/utils/kubernetesTemplates/zeppelinService.yaml

@@ -8,7 +8,7 @@ spec:
     - name: http
       protocol: TCP
       port: 80
-      targetPort: 80
+      targetPort: 8080
     - name: rpc
       protocol: TCP
       port: 12320

api/utils/zeppelinAPI.py

@@ -149,8 +149,11 @@ def restartInterpreter(self, interpreterName: str):
         return self.__parseResponse(response)
 
     def healthCheck(self):
-        response = requests.get(f"{self.ZEPPELIN_ADDR}/{ZEPPELIN_VERSION_ENDPOINT}")
-        return self.__parseResponse(response)
+        try:
+            response = requests.get(f"{self.ZEPPELIN_ADDR}/{ZEPPELIN_VERSION_ENDPOINT}")
+            return self.__parseResponse(response)
+        except Exception as ex:
+            return False
 
     def __parseResponse(self, response):
         """

cuelake.yaml

@@ -117,7 +117,7 @@ spec:
             path: nginx.conf
       containers:
       - name: zeppelin-server
-        image: cuebook/zeppelin-server:0.9.0
+        image: cuebook/zeppelin-server-lite:0.2
         command: ["sh", "-c"]
         args:
           - curl https://raw.githubusercontent.com/cuebook/cuelake/main/zeppelinConf/interpreter.json -o $(ZEPPELIN_HOME)/conf/interpreter.json && curl https://raw.githubusercontent.com/cuebook/cuelake/main/zeppelinConf/zeppelin-env.sh -o $(ZEPPELIN_HOME)/conf/zeppelin-env.sh && curl https://raw.githubusercontent.com/cuebook/cuelake/main/zeppelinConf/zeppelin-site.xml -o $(ZEPPELIN_HOME)/conf/zeppelin-site.xml && $(ZEPPELIN_HOME)/bin/zeppelin.sh
@@ -211,7 +211,7 @@ metadata:
   name: zeppelin-server-role
 rules:
 - apiGroups: [""]
-  resources: ["pods", "services", "configmaps"]
+  resources: ["pods", "pods/exec", "services", "configmaps"]
   verbs: ["create", "get", "update", "patch", "list", "delete", "watch"]
 - apiGroups: ["rbac.authorization.k8s.io"]
   resources: ["roles", "rolebindings"]
@@ -300,16 +300,27 @@ spec:
       labels:
         app.kubernetes.io/name: lakehouse
     spec:
+      serviceAccountName: zeppelin-server
       volumes:
       - name: lakehouse-db-volume
         persistentVolumeClaim:
           claimName: lakehouse-db-volume-pvc
       containers:
       - name: lakehouse
-        image: cuebook/lakehouse:latest
+        image: cuebook/lakehouse:0.2
+        resources:
+          requests:
+            memory: "2560Mi"
+            cpu: "250m"
         volumeMounts:
         - name: lakehouse-db-volume     # configure this to persist db
           mountPath: /code/db
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
         envFrom:
         - configMapRef:
             name: cuelake-conf-map