resource-managers\pom.xml
org.apache.spark.deploy.SparkSubmit.scala
org.apache.spark.deploy.k8s.submit.Client.scala
org.apache.spark.deploy.k8s.submit.submitsteps.KubernetesDriverSpec.scala
org.apache.spark.deploy.k8s.submit.Client.scala
debug：
0：
1：
2：
3：
4：
5：
結果：
{
"kind": "Pod",
"apiVersion": "v1",
"metadata": {
"name": "iot-stream-analysis-driver-104",
"namespace": "spark-cluster",
"selfLink": "/api/v1/namespaces/spark-cluster/pods/iot-stream-analysis-driver-104",
"uid": "9897bb77-b4c6-11e8-910e-0050568848e8",
"resourceVersion": "32841344",
"creationTimestamp": "2018-09-10T06:56:09Z",
"labels": {
"spark-app-selector": "spark-a680ef1ed8364d3b96dfbc9e5d3f1d83",
"spark-role": "driver"
},
"annotations": {
"spark-app-name": "spark-job-on-k8s-104"
}
},
"spec": {
"volumes": [
{
"name": "spark-local-dir-0-spark-1ce8faad-1066-4386-baa0-f8a9de2231f3",
"emptyDir": {}
},
{
"name": "spark-init-properties",
"configMap": {
"name": "spark-job-on-k8s-104-1536562620604-init-config",
"items": [
{
"key": "download-submitted-files",
"path": "spark-init.properties"
}
],
"defaultMode": 420
}
},
{
"name": "download-jars-volume",
"emptyDir": {}
},
{
"name": "download-files",
"emptyDir": {}
},
{
"name": "spark-init-secret",
"secret": {
"secretName": "spark-job-on-k8s-104-1536562620604-init-secret",
"defaultMode": 420
}
},
{
"name": "spark-token-kpkrp",
"secret": {
"secretName": "spark-token-kpkrp",
"defaultMode": 420
}
}
],
"initContainers": [
{
"name": "spark-init",
"image": "registry.dev161.com:5000/iot/spark-init:v2.2.0-kubernetes-0.5.1",
"args": [
"/etc/spark-init/spark-init.properties"
],
"resources": {},
"volumeMounts": [
{
"name": "spark-init-properties",
"mountPath": "/etc/spark-init"
},
{
"name": "download-jars-volume",
"mountPath": "/var/spark-data/spark-jars"
},
{
"name": "download-files",
"mountPath": "/var/spark-data/spark-files"
},
{
"name": "spark-init-secret",
"mountPath": "/mnt/secrets/spark-init"
},
{
"name": "spark-token-kpkrp",
"readOnly": true,
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
}
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "Always"
}
],
"containers": [
{
"name": "spark-kubernetes-driver",
"image": "registry.dev161.com:5000/iot/spark-driver:v2.2.0-kubernetes-0.5.1",
"env": [
{
"name": "SPARK_DRIVER_MEMORY",
"value": "1024m"
},
{
"name": "SPARK_DRIVER_CLASS",
"value": "com.inspur.iot.RuleEngine"
},
{
"name": "SPARK_DRIVER_ARGS",
"value": "--base64=true --rule=c2VsZWN0IHRpbWVTdGFtcCBBcyBrZXksIGNvbmNhdF93cygifCIsIHN0YXRlLnJlcG9ydGVkLnRlbXBlcmF0dXJlLCBjbGllbnRUb2tlbikgYXMgdmFsdWUgZnJvbSB0b3BpY3M= --sample={\"timeStamp\":1531381822,\"clientToken\":\"clientId_lamp\",\"state\":{\"reported\":{\"temperature\":23}}} --source-type=kafka --source={\"kafka.bootstrap.servers\":\"insight-kafka-svc.default:9092\",\"subscribe\":\"sensor\"} --sink-type=console --verbose"
},
{
"name": "SPARK_DRIVER_BIND_ADDRESS",
"valueFrom": {
"fieldRef": {
"apiVersion": "v1",
"fieldPath": "status.podIP"
}
}
},
{
"name": "SPARK_MOUNTED_CLASSPATH",
"value": "/var/spark-data/spark-jars/iot-stream-app-1.3-SNAPSHOT.jar"
},
{
"name": "SPARK_MOUNTED_FILES_DIR",
"value": "/var/spark-data/spark-files"
},
{
"name": "SPARK_JAVA_OPT_0",
"value": "-Dspark.kubernetes.svc.domain=svc.dev161"
},
{
"name": "SPARK_JAVA_OPT_1",
"value": "-Dspark.kubernetes.initcontainer.executor.configmapkey=download-submitted-files"
},
{
"name": "SPARK_JAVA_OPT_2",
"value": "-Dspark.app.id=spark-a680ef1ed8364d3b96dfbc9e5d3f1d83"
},
{
"name": "SPARK_JAVA_OPT_3",
"value": "-Dspark.submit.deployMode=cluster"
},
{
"name": "SPARK_JAVA_OPT_4",
"value": "-Dspark.kubernetes.authenticate.driver.serviceAccountName=spark"
},
{
"name": "SPARK_JAVA_OPT_5",
"value": "-Dspark.kubernetes.resourceStagingServer.uri=http://10.110.20.161:31000"
},
{
"name": "SPARK_JAVA_OPT_6",
"value": "-Dspark.app.name=spark-job-on-k8s-104"
},
{
"name": "SPARK_JAVA_OPT_7",
"value": "-Dspark.jars=/var/spark-data/spark-jars/iot-stream-app-1.3-SNAPSHOT.jar"
},
{
"name": "SPARK_JAVA_OPT_8",
"value": "-Dspark.driver.host=spark-job-on-k8s-104-1536562620604-driver-svc.spark-cluster.svc.cluster.local"
},
{
"name": "SPARK_JAVA_OPT_9",
"value": "-Dspark.local.dir=/mnt/tmp/spark-local/spark-1ce8faad-1066-4386-baa0-f8a9de2231f3"
},
{
"name": "SPARK_JAVA_OPT_10",
"value": "-Dspark.kubernetes.initcontainer.docker.image=registry.dev161.com:5000/iot/spark-init:v2.2.0-kubernetes-0.5.1"
},
{
"name": "SPARK_JAVA_OPT_11",
"value": "-Dspark.kubernetes.initcontainer.executor.configmapname=spark-job-on-k8s-104-1536562620604-init-config"
},
{
"name": "SPARK_JAVA_OPT_12",
"value": "-Dspark.kubernetes.initcontainer.executor.stagingServerSecret.name=spark-job-on-k8s-104-1536562620604-init-secret"
},
{
"name": "SPARK_JAVA_OPT_13",
"value": "-Dspark.driver.cores=1"
},
{
"name": "SPARK_JAVA_OPT_14",
"value": "-Dspark.kubernetes.docker.image.pullPolicy=Always"
},
{
"name": "SPARK_JAVA_OPT_15",
"value": "-Dspark.kubernetes.driver.docker.image=registry.dev161.com:5000/iot/spark-driver:v2.2.0-kubernetes-0.5.1"
},
{
"name": "SPARK_JAVA_OPT_16",
"value": "-Dspark.kubernetes.driver.nodeport=31202"
},
{
"name": "SPARK_JAVA_OPT_17",
"value": "-Dspark.driver.blockManager.port=7079"
},
{
"name": "SPARK_JAVA_OPT_18",
"value": "-Dspark.kubernetes.namespace=spark-cluster"
},
{
"name": "SPARK_JAVA_OPT_19",
"value": "-Dspark.kubernetes.driver.pod.name=iot-stream-analysis-driver-104"
},
{
"name": "SPARK_JAVA_OPT_20",
"value": "-Dspark.driver.memory=1024m"
},
{
"name": "SPARK_JAVA_OPT_21",
"value": "-Dspark.kubernetes.submission.waitAppCompletion=true"
},
{
"name": "SPARK_JAVA_OPT_22",
"value": "-Dspark.driver.port=7078"
},
{
"name": "SPARK_JAVA_OPT_23",
"value": "-Dspark.kubernetes.initcontainer.executor.stagingServerSecret.mountDir=/mnt/secrets/spark-init"
},
{
"name": "SPARK_JAVA_OPT_24",
"value": "-Dspark.master=k8s://http://10.110.20.161:8080"
},
{
"name": "SPARK_JAVA_OPT_25",
"value": "-Dspark.executor.memory=1024m"
},
{
"name": "SPARK_JAVA_OPT_26",
"value": "-Dspark.kubernetes.executor.podNamePrefix=spark-job-on-k8s-104-1536562620604"
},
{
"name": "SPARK_JAVA_OPT_27",
"value": "-Dspark.ui.enabled=false"
},
{
"name": "SPARK_JAVA_OPT_28",
"value": "-Dspark.executor.instances=1"
},
{
"name": "SPARK_JAVA_OPT_29",
"value": "-Dspark.kubernetes.executor.docker.image=registry.dev161.com:5000/iot/spark-executor:v2.2.0-kubernetes-0.5.1"
},
{
"name": "SPARK_JAVA_OPT_30",
"value": "-Dspark.executor.cores=1"
}
],
"resources": {
"limits": {
"memory": "1408Mi"
},
"requests": {
"cpu": "1",
"memory": "1Gi"
}
},
"volumeMounts": [
{
"name": "spark-local-dir-0-spark-1ce8faad-1066-4386-baa0-f8a9de2231f3",
"mountPath": "/mnt/tmp/spark-local/spark-1ce8faad-1066-4386-baa0-f8a9de2231f3"
},
{
"name": "download-jars-volume",
"mountPath": "/var/spark-data/spark-jars"
},
{
"name": "download-files",
"mountPath": "/var/spark-data/spark-files"
},
{
"name": "spark-token-kpkrp",
"readOnly": true,
"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount"
}
],
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File",
"imagePullPolicy": "Always"
}
],
"restartPolicy": "Never",
"terminationGracePeriodSeconds": 30,
"dnsPolicy": "ClusterFirst",
"serviceAccountName": "spark",
"serviceAccount": "spark",
"nodeName": "master1",
"securityContext": {},
"schedulerName": "default-scheduler"
},
"status": {
"phase": "Running",
"conditions": [
{
"type": "Initialized",
"status": "True",
"lastProbeTime": null,
"lastTransitionTime": "2018-09-10T06:56:18Z"
},
{
"type": "Ready",
"status": "True",
"lastProbeTime": null,
"lastTransitionTime": "2018-09-10T06:56:19Z"
},
{
"type": "PodScheduled",
"status": "True",
"lastProbeTime": null,
"lastTransitionTime": "2018-09-10T06:56:09Z"
}
],
"hostIP": "10.110.20.161",
"podIP": "161.161.104.93",
"startTime": "2018-09-10T06:56:09Z",
"initContainerStatuses": [
{
"name": "spark-init",
"state": {
"terminated": {
"exitCode": 0,
"reason": "Completed",
"startedAt": "2018-09-10T06:56:13Z",
"finishedAt": "2018-09-10T06:56:18Z",
"containerID": "docker://c127670c8e9c1157b00ce29145dff658a89576dc3d785355c6f4dce00df1606f"
}
},
"lastState": {},
"ready": true,
"restartCount": 0,
"image": "registry.dev161.com:5000/iot/spark-init:v2.2.0-kubernetes-0.5.1",
"imageID": "docker-pullable://registry.dev161.com:5000/iot/spark-init@sha256:adca6ed85868d821e3f63e49b7d44765954bcf5e4131a4d419cd9f4dc2235401",
"containerID": "docker://c127670c8e9c1157b00ce29145dff658a89576dc3d785355c6f4dce00df1606f"
}
],
"containerStatuses": [
{
"name": "spark-kubernetes-driver",
"state": {
"running": {
"startedAt": "2018-09-10T06:56:19Z"
}
},
"lastState": {},
"ready": true,
"restartCount": 0,
"image": "registry.dev161.com:5000/iot/spark-driver:v2.2.0-kubernetes-0.5.1",
"imageID": "docker-pullable://registry.dev161.com:5000/iot/spark-driver@sha256:6cf4ab9fd7ff1c21ddd1d66feb76107928c844cfaae912172bc618734942ff80",
"containerID": "docker://6cb1f8004505aa3440827691720bc35eff1a64936da8fd4d71e82afade29e05d"
}
],
"qosClass": "Burstable"
}
}{
"kind": "ConfigMap",
"apiVersion": "v1",
"metadata": {
"name": "spark-job-on-k8s-104-1536562620604-init-config",
"namespace": "spark-cluster",
"selfLink": "/api/v1/namespaces/spark-cluster/configmaps/spark-job-on-k8s-104-1536562620604-init-config",
"uid": "99991bea-b4c6-11e8-910e-0050568848e8",
"resourceVersion": "32841315",
"creationTimestamp": "2018-09-10T06:56:10Z",
"ownerReferences": [
{
"apiVersion": "v1",
"kind": "Pod",
"name": "iot-stream-analysis-driver-104",
"uid": "9897bb77-b4c6-11e8-910e-0050568848e8",
"controller": true
}
]
},
"data": {
"download-submitted-files": "#Java properties built from Kubernetes config map with name: spark-job-on-k8s-104-1536562620604-init-config and config map key: $configMapKey\r\n#Mon Sep 10 14:57:08 CST 2018\r\nspark.kubernetes.initcontainer.downloadJarsSecretLocation=/mnt/secrets/spark-init/downloadSubmittedJarsSecret\r\nspark.kubernetes.initcontainer.downloadFilesSecretLocation=/mnt/secrets/spark-init/downloadSubmittedFilesSecret\r\nspark.kubernetes.initcontainer.downloadJarsResourceIdentifier=a6c0295f-495a-4967-856e-7e2e607cb0f8\r\nspark.kubernetes.initcontainer.downloadFilesResourceIdentifier=d673b9f6-acd0-4cbd-9851-fc5a9ae1c760\r\nspark.ssl.kubernetes.resourceStagingServer.enabled=false\r\nspark.kubernetes.resourceStagingServer.uri=http\\://10.110.20.161\\:31000\r\nspark.kubernetes.mountdependencies.filesDownloadDir=/var/spark-data/spark-files\r\nspark.kubernetes.mountdependencies.jarsDownloadDir=/var/spark-data/spark-jars\r\n"
}
}{
"kind": "Service",
"apiVersion": "v1",
"metadata": {
"name": "spark-job-on-k8s-104-1536562620604-driver-svc",
"namespace": "spark-cluster",
"selfLink": "/api/v1/namespaces/spark-cluster/services/spark-job-on-k8s-104-1536562620604-driver-svc",
"uid": "99892978-b4c6-11e8-910e-0050568848e8",
"resourceVersion": "32841312",
"creationTimestamp": "2018-09-10T06:56:10Z",
"ownerReferences": [
{
"apiVersion": "v1",
"kind": "Pod",
"name": "iot-stream-analysis-driver-104",
"uid": "9897bb77-b4c6-11e8-910e-0050568848e8",
"controller": true
}
]
},
"spec": {
"ports": [
{
"name": "driver-rpc-port",
"protocol": "TCP",
"port": 7078,
"targetPort": 7078
},
{
"name": "blockmanager",
"protocol": "TCP",
"port": 7079,
"targetPort": 7079
}
],
"selector": {
"spark-app-selector": "spark-a680ef1ed8364d3b96dfbc9e5d3f1d83",
"spark-role": "driver"
},
"clusterIP": "None",
"type": "ClusterIP",
"sessionAffinity": "None"
},
"status": {
"loadBalancer": {}
}
}container:
init-container:
[
{
"Id": "d11c77e3981cd932284d5503f3ea2450a5ae65736cd34d7daa264f33d45b0957",
"Created": "2018-09-11T03:47:40.443123953Z",
"Path": "/opt/entrypoint.sh",
"Args": [
"/opt/spark/bin/spark-class",
"org.apache.spark.deploy.rest.k8s.KubernetesSparkDependencyDownloadInitContainer",
"/etc/spark-init/spark-init.properties"
],
"State": {
"Status": "exited",
"Running": false,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 0,
"ExitCode": 0,
"Error": "",
"StartedAt": "2018-09-11T03:47:40.925573911Z",
"FinishedAt": "2018-09-11T03:47:45.500154065Z"
},
"Image": "sha256:45b585586ede8839199405d7cb01fede31f5dc9fdeda7d7cf4b0aaa24050b4db",
"ResolvConfPath": "/var/lib/docker/containers/d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b/hostname",
"HostsPath": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts",
"LogPath": "/var/lib/docker/containers/d11c77e3981cd932284d5503f3ea2450a5ae65736cd34d7daa264f33d45b0957/d11c77e3981cd932284d5503f3ea2450a5ae65736cd34d7daa264f33d45b0957-json.log",
"Name": "/k8s_spark-init_iot-stream-analysis-driver-100_spark-cluster_64491f22-b575-11e8-910e-0050568848e8_0",
"RestartCount": 0,
"Driver": "overlay",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~configmap/spark-init-properties:/etc/spark-init",
"/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume:/var/spark-data/spark-jars",
"/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files:/var/spark-data/spark-files",
"/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-init-secret:/mnt/secrets/spark-init",
"/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-token-kpkrp:/var/run/secrets/kubernetes.io/serviceaccount:ro",
"/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts:/etc/hosts",
"/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/containers/spark-init/206592cf:/dev/termination-log"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {
"max-file": "5",
"max-size": "50m"
}
},
"NetworkMode": "container:d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b",
"PortBindings": null,
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "container:d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 999,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": [
"seccomp=unconfined"
],
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 2,
"Memory": 0,
"NanoCpus": 0,
"CgroupParent": "kubepods-burstable-pod64491f22_b575_11e8_910e_0050568848e8.slice",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 0,
"MemorySwappiness": -1,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0
},
"GraphDriver": {
"Name": "overlay",
"Data": {
"LowerDir": "/var/lib/docker/overlay/24dfa74479a001ecabd6d8ab9f61337a898ad9ab78e914b5ab17096c357eaf5e/root",
"MergedDir": "/var/lib/docker/overlay/c31c6eec68c6405dd62c8dd7dfea4707cebca534df2dced5f0b158969c0dd7e6/merged",
"UpperDir": "/var/lib/docker/overlay/c31c6eec68c6405dd62c8dd7dfea4707cebca534df2dced5f0b158969c0dd7e6/upper",
"WorkDir": "/var/lib/docker/overlay/c31c6eec68c6405dd62c8dd7dfea4707cebca534df2dced5f0b158969c0dd7e6/work"
}
},
"Mounts": [
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts",
"Destination": "/etc/hosts",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/containers/spark-init/206592cf",
"Destination": "/dev/termination-log",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~configmap/spark-init-properties",
"Destination": "/etc/spark-init",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume",
"Destination": "/var/spark-data/spark-jars",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files",
"Destination": "/var/spark-data/spark-files",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-init-secret",
"Destination": "/mnt/secrets/spark-init",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-token-kpkrp",
"Destination": "/var/run/secrets/kubernetes.io/serviceaccount",
"Mode": "ro",
"RW": false,
"Propagation": ""
}
],
"Config": {
"Hostname": "iot-stream-analysis-driver-100",
"Domainname": "",
"User": "0",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"KUBERNETES_SERVICE_PORT=443",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.52.23:4040",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT=tcp://161.161.13.152:4040",
"KUBERNETES_PORT_443_TCP_PORT=443",
"KUBERNETES_PORT_443_TCP_ADDR=161.161.0.1",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_HOST=161.161.52.23",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT=4040",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.13.152:4040",
"KUBERNETES_SERVICE_HOST=161.161.0.1",
"KUBERNETES_SERVICE_PORT_HTTPS=443",
"KUBERNETES_PORT=tcp://161.161.0.1:443",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.13.152",
"KUBERNETES_PORT_443_TCP=tcp://161.161.0.1:443",
"KUBERNETES_PORT_443_TCP_PROTO=tcp",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT=tcp://161.161.52.23:4040",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.52.23",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_HOST=161.161.13.152",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT=4040",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin",
"LANG=C.UTF-8",
"JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk",
"JAVA_VERSION=8u131",
"JAVA_ALPINE_VERSION=8.131.11-r2",
"SPARK_HOME=/opt/spark"
],
"Cmd": [
"/etc/spark-init/spark-init.properties"
],
"Healthcheck": {
"Test": [
"NONE"
]
},
"Image": "registry.dev161.com:5000/iot/spark-init@sha256:adca6ed85868d821e3f63e49b7d44765954bcf5e4131a4d419cd9f4dc2235401",
"Volumes": null,
"WorkingDir": "/opt/spark/work-dir",
"Entrypoint": [
"/opt/entrypoint.sh",
"/opt/spark/bin/spark-class",
"org.apache.spark.deploy.rest.k8s.KubernetesSparkDependencyDownloadInitContainer"
],
"OnBuild": null,
"Labels": {
"annotation.io.kubernetes.container.hash": "7757e3a7",
"annotation.io.kubernetes.container.restartCount": "0",
"annotation.io.kubernetes.container.terminationMessagePath": "/dev/termination-log",
"annotation.io.kubernetes.container.terminationMessagePolicy": "File",
"annotation.io.kubernetes.pod.terminationGracePeriod": "30",
"io.kubernetes.container.logpath": "/var/log/pods/64491f22-b575-11e8-910e-0050568848e8/spark-init_0.log",
"io.kubernetes.container.name": "spark-init",
"io.kubernetes.docker.type": "container",
"io.kubernetes.pod.name": "iot-stream-analysis-driver-100",
"io.kubernetes.pod.namespace": "spark-cluster",
"io.kubernetes.pod.uid": "64491f22-b575-11e8-910e-0050568848e8",
"io.kubernetes.sandbox.id": "d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": null,
"SandboxKey": "",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {}
}
}
]driver:
[
{
"Id": "2e96d8a55415dc58420d346243b83968a4c1a1098000ad76af9152f2ba904a0c",
"Created": "2018-09-11T03:47:46.771489986Z",
"Path": "/opt/entrypoint.sh",
"Args": [
"/bin/sh",
"-c",
"SPARK_CLASSPATH=\"${SPARK_HOME}/jars/*\" && env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\\(.*\\)/\\1/g' > /tmp/java_opts.txt && readarray -t SPARK_DRIVER_JAVA_OPTS < /tmp
/java_opts.txt && if ! [ -z ${SPARK_MOUNTED_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_SUBMIT_EXTRA_CLASSPATH+x} ];
then SPARK_CLASSPATH=\"$SPARK_SUBMIT_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH\";
fi && if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_DIR/.\" .; fi && if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR} ]; then cp -R \"$SPARK_MOUNTED_FI
LES_FROM_SECRET_DIR/.\" .; fi && ${JAVA_HOME}/bin/java \"${SPARK_DRIVER_JAVA_OPTS[@]}\" -cp $SPARK_CLASSPATH -Xms$SPARK_DRIVER_MEMORY -Xmx$SPARK_DRIVER_MEMORY -Dspark.driver.bindAddress
=$SPARK_DRIVER_BIND_ADDRESS $SPARK_DRIVER_CLASS $SPARK_DRIVER_ARGS"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 832405,
"ExitCode": 0,
"Error": "",
"StartedAt": "2018-09-11T03:47:47.066061613Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:4ba1d64922f4ccfb682ca09d7d716252f41c55a650884b796598bafb03174805",
"ResolvConfPath": "/var/lib/docker/containers/d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b/hostname",
"HostsPath": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts",
"LogPath": "/var/lib/docker/containers/2e96d8a55415dc58420d346243b83968a4c1a1098000ad76af9152f2ba904a0c/2e96d8a55415dc58420d346243b83968a4c1a1098000ad76af9152f2ba904a0c-json.log",
"Name": "/k8s_spark-kubernetes-driver_iot-stream-analysis-driver-100_spark-cluster_64491f22-b575-11e8-910e-0050568848e8_0",
"RestartCount": 0,
"Driver": "overlay",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/spark-local-dir-0-spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58:/mnt/tmp/spark-local
/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58",
"/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume:/var/spark-data/spark-jars",
"/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files:/var/spark-data/spark-files",
"/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-token-kpkrp:/var/run/secrets/kubernetes.io/serviceaccount:ro",
"/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts:/etc/hosts",
"/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/containers/spark-kubernetes-driver/65c96924:/dev/termination-log"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {
"max-file": "5",
"max-size": "50m"
}
},
"NetworkMode": "container:d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b",
"PortBindings": null,
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "container:d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 936,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": [
"seccomp=unconfined"
],
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 1024,
"Memory": 1476395008,
"NanoCpus": 0,
"CgroupParent": "kubepods-burstable-pod64491f22_b575_11e8_910e_0050568848e8.slice",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 2952790016,
"MemorySwappiness": -1,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0
},
"GraphDriver": {
"Name": "overlay",
"Data": {
"LowerDir": "/var/lib/docker/overlay/e7d727c6b28f718f78192d5bc76f24dffcf4603fba6e6173bded398b95d4c578/root",
"MergedDir": "/var/lib/docker/overlay/2644946a6d486a66c93455eb2220b15227386d9329576df23568bc0f5221e6de/merged",
"UpperDir": "/var/lib/docker/overlay/2644946a6d486a66c93455eb2220b15227386d9329576df23568bc0f5221e6de/upper",
"WorkDir": "/var/lib/docker/overlay/2644946a6d486a66c93455eb2220b15227386d9329576df23568bc0f5221e6de/work"
}
},
"Mounts": [
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/spark-token-kpkrp",
"Destination": "/var/run/secrets/kubernetes.io/serviceaccount",
"Mode": "ro",
"RW": false,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/etc-hosts",
"Destination": "/etc/hosts",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/containers/spark-kubernetes-driver/65c96924",
"Destination": "/dev/termination-log",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/spark-local-dir-0-spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58",
"Destination": "/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume",
"Destination": "/var/spark-data/spark-jars",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/64491f22-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files",
"Destination": "/var/spark-data/spark-files",
"Mode": "",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "iot-stream-analysis-driver-100",
"Domainname": "",
"User": "0",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"SPARK_DRIVER_BIND_ADDRESS=161.161.104.67",
"SPARK_JAVA_OPT_9=-Dspark.kubernetes.initcontainer.docker.image=registry.dev161.com:5000/iot/spark-init:v2.2.0-kubernetes-0.5.1",
"SPARK_JAVA_OPT_10=-Dspark.kubernetes.driver.pod.name=iot-stream-analysis-driver-100",
"SPARK_JAVA_OPT_13=-Dspark.kubernetes.docker.image.pullPolicy=Always",
"SPARK_JAVA_OPT_14=-Dspark.kubernetes.driver.docker.image=registry.dev161.com:5000/iot/spark-driver:v2.2.0-kubernetes-0.5.1",
"SPARK_JAVA_OPT_29=-Dspark.kubernetes.executor.docker.image=registry.dev161.com:5000/iot/spark-executor:v2.2.0-kubernetes-0.5.1",
"SPARK_DRIVER_MEMORY=1024m",
"SPARK_JAVA_OPT_1=-Dspark.kubernetes.svc.domain=svc.dev161",
"SPARK_JAVA_OPT_5=-Dspark.local.dir=/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58",
"SPARK_JAVA_OPT_11=-Dspark.app.name=spark-job-on-k8s-100",
"SPARK_JAVA_OPT_19=-Dspark.kubernetes.executor.podNamePrefix=spark-job-on-k8s-100-1536637851433",
"SPARK_JAVA_OPT_22=-Dspark.driver.port=7078",
"SPARK_JAVA_OPT_25=-Dspark.executor.memory=1024m",
"SPARK_MOUNTED_FILES_DIR=/var/spark-data/spark-files",
"SPARK_JAVA_OPT_0=-Dspark.driver.host=spark-job-on-k8s-100-1536637851433-driver-svc.spark-cluster.svc.dev161",
"SPARK_JAVA_OPT_3=-Dspark.submit.deployMode=cluster",
"SPARK_JAVA_OPT_7=-Dspark.kubernetes.resourceStagingServer.uri=http://10.110.20.161:31000",
"SPARK_JAVA_OPT_27=-Dspark.app.id=spark-1619ed3f66b04bf9a54a7230a1f551a4",
"SPARK_JAVA_OPT_20=-Dspark.driver.memory=1024m",
"SPARK_JAVA_OPT_21=-Dspark.kubernetes.submission.waitAppCompletion=true",
"SPARK_JAVA_OPT_26=-Dspark.ui.enabled=false",
"SPARK_MOUNTED_CLASSPATH=/var/spark-data/spark-jars/iot-stream-app-1.3-SNAPSHOT.jar",
"SPARK_JAVA_OPT_2=-Dspark.kubernetes.initcontainer.executor.configmapkey=download-submitted-files",
"SPARK_JAVA_OPT_18=-Dspark.kubernetes.initcontainer.executor.stagingServerSecret.name=spark-job-on-k8s-100-1536637851433-init-secret",
"SPARK_JAVA_OPT_23=-Dspark.kubernetes.initcontainer.executor.stagingServerSecret.mountDir=/mnt/secrets/spark-init",
"SPARK_JAVA_OPT_28=-Dspark.executor.instances=1",
"SPARK_JAVA_OPT_30=-Dspark.executor.cores=1",
"SPARK_DRIVER_ARGS=--base64=true --rule=c2VsZWN0IHRpbWVTdGFtcCBBcyBrZXksIGNvbmNhdF93cygifCIsIHN0YXRlLnJlcG9ydGVkLnRlbXBlcmF0dXJlLCBjbGllbnRUb2tlbikgYXMgdmFsdWUgZnJvbSB0b3BpY
3M= --sample={\"timeStamp\":1531381822,\"clientToken\":\"clientId_lamp\",\"state\":{\"reported\":{\"temperature\":23}}} --source-type=kafka --source={\"kafka.bootstrap.servers\":\"fnd-kafka
-svc.default:9092\",\"subscribe\":\"sensor\"} --sink-type=console --verbose",
"SPARK_JAVA_OPT_8=-Dspark.jars=/var/spark-data/spark-jars/iot-stream-app-1.3-SNAPSHOT.jar",
"SPARK_JAVA_OPT_12=-Dspark.driver.cores=1",
"SPARK_JAVA_OPT_15=-Dspark.kubernetes.driver.nodeport=31202",
"SPARK_JAVA_OPT_17=-Dspark.kubernetes.namespace=spark-cluster",
"SPARK_JAVA_OPT_4=-Dspark.kubernetes.authenticate.driver.serviceAccountName=spark",
"SPARK_JAVA_OPT_6=-Dspark.kubernetes.initcontainer.executor.configmapname=spark-job-on-k8s-100-1536637851433-init-config",
"SPARK_JAVA_OPT_16=-Dspark.driver.blockManager.port=7079",
"SPARK_DRIVER_CLASS=com.inspur.iot.RuleEngine",
"SPARK_JAVA_OPT_24=-Dspark.master=k8s://http://10.110.20.161:8080",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_HOST=161.161.13.152",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040",
"KUBERNETES_SERVICE_PORT_HTTPS=443",
"KUBERNETES_PORT_443_TCP_PROTO=tcp",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.52.23",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.13.152:4040",
"KUBERNETES_PORT=tcp://161.161.0.1:443",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT=4040",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT=tcp://161.161.13.152:4040",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp",
"KUBERNETES_SERVICE_PORT=443",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_HOST=161.161.52.23",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT=tcp://161.161.52.23:4040",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT=4040",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.13.152",
"KUBERNETES_SERVICE_HOST=161.161.0.1",
"KUBERNETES_PORT_443_TCP=tcp://161.161.0.1:443",
"KUBERNETES_PORT_443_TCP_PORT=443",
"KUBERNETES_PORT_443_TCP_ADDR=161.161.0.1",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.52.23:4040",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin",
"LANG=C.UTF-8",
"JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk",
"JAVA_VERSION=8u131",
"JAVA_ALPINE_VERSION=8.131.11-r2",
"SPARK_HOME=/opt/spark"
],
"Cmd": [
"/bin/sh",
"-c",
"SPARK_CLASSPATH=\"${SPARK_HOME}/jars/*\" && env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\\(.*\\)/\\1/g' > /tmp/java_opts.txt && readarray -t SPARK_DRIVER_JAVA_OPTS <
/tmp/java_opts.txt && if ! [ -z ${SPARK_MOUNTED_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_SUBMIT_EXTRA_CLASSPATH+x}
]; then SPARK_CLASSPATH=\"$SPARK_SUBMIT_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH
\"; fi && if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_DIR/.\" .; fi && if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR} ]; then cp -R \"$SPARK_MOUNTE
D_FILES_FROM_SECRET_DIR/.\" .; fi && ${JAVA_HOME}/bin/java \"${SPARK_DRIVER_JAVA_OPTS[@]}\" -cp $SPARK_CLASSPATH -Xms$SPARK_DRIVER_MEMORY -Xmx$SPARK_DRIVER_MEMORY -Dspark.driver.bindAdd
ress=$SPARK_DRIVER_BIND_ADDRESS $SPARK_DRIVER_CLASS $SPARK_DRIVER_ARGS"
],
"Healthcheck": {
"Test": [
"NONE"
]
},
"ArgsEscaped": true,
"Image": "registry.dev161.com:5000/iot/spark-driver@sha256:6cf4ab9fd7ff1c21ddd1d66feb76107928c844cfaae912172bc618734942ff80",
"Volumes": null,
"WorkingDir": "/opt/spark/work-dir",
"Entrypoint": [
"/opt/entrypoint.sh"
],
"OnBuild": null,
"Labels": {
"annotation.io.kubernetes.container.hash": "57d51e36",
"annotation.io.kubernetes.container.restartCount": "0",
"annotation.io.kubernetes.container.terminationMessagePath": "/dev/termination-log",
"annotation.io.kubernetes.container.terminationMessagePolicy": "File",
"annotation.io.kubernetes.pod.terminationGracePeriod": "30",
"io.kubernetes.container.logpath": "/var/log/pods/64491f22-b575-11e8-910e-0050568848e8/spark-kubernetes-driver_0.log",
"io.kubernetes.container.name": "spark-kubernetes-driver",
"io.kubernetes.docker.type": "container",
"io.kubernetes.pod.name": "iot-stream-analysis-driver-100",
"io.kubernetes.pod.namespace": "spark-cluster",
"io.kubernetes.pod.uid": "64491f22-b575-11e8-910e-0050568848e8",
"io.kubernetes.sandbox.id": "d16c636974bbbc1e92a242dc0d8b43472c2816c20bdcf8a3eca0789cb3187c8b"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": null,
"SandboxKey": "",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {}
}
}
]executor:
[
{
"Id": "37f1bce3f1295156a691768c4d385903f7cb978d35e63b3e5931d32e2802c322",
"Created": "2018-09-11T03:47:55.861508477Z",
"Path": "/opt/entrypoint.sh",
"Args": [
"/bin/sh",
"-c",
"SPARK_CLASSPATH=\"${SPARK_HOME}/jars/*\" && env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\\(.*\\)/\\1/g' > /tmp/java_opts.txt && readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt && if ! [ -z ${SPARK_MOUNTED_CLASSPATH}+x} ]; then SPARK_CLASSPATH=\"$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXECUTOR_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXECUTOR_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_DIR/.\" .; fi && if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/.\" .; fi && ${JAVA_HOME}/bin/java \"${SPARK_EXECUTOR_JAVA_OPTS[@]}\" -Dspark.executor.port=$SPARK_EXECUTOR_PORT -Xms$SPARK_EXECUTOR_MEMORY -Xmx$SPARK_EXECUTOR_MEMORY -cp $SPARK_CLASSPATH org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url $SPARK_DRIVER_URL --executor-id $SPARK_EXECUTOR_ID --cores $SPARK_EXECUTOR_CORES --app-id $SPARK_APPLICATION_ID --hostname $SPARK_EXECUTOR_POD_IP"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 11397,
"ExitCode": 0,
"Error": "",
"StartedAt": "2018-09-11T03:47:55.982525484Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:c45bb8d3fb02e8684a36f5cc02d00548ddeffe1102bd8e80e330ef715c470de9",
"ResolvConfPath": "/var/lib/docker/containers/0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b/hostname",
"HostsPath": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/etc-hosts",
"LogPath": "/var/lib/docker/containers/37f1bce3f1295156a691768c4d385903f7cb978d35e63b3e5931d32e2802c322/37f1bce3f1295156a691768c4d385903f7cb978d35e63b3e5931d32e2802c322-json.log",
"Name": "/k8s_executor_spark-job-on-k8s-100-1536637851433-exec-1_spark-cluster_74a94609-b575-11e8-910e-0050568848e8_0",
"RestartCount": 0,
"Driver": "overlay",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/spark-local-dir-0-spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58:/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58",
"/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume:/var/spark-data/spark-jars",
"/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files:/var/spark-data/spark-files",
"/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/default-token-jxg4k:/var/run/secrets/kubernetes.io/serviceaccount:ro",
"/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/etc-hosts:/etc/hosts",
"/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/containers/executor/664b5635:/dev/termination-log"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {
"max-file": "5",
"max-size": "50m"
}
},
"NetworkMode": "container:0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b",
"PortBindings": null,
"RestartPolicy": {
"Name": "",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"CapAdd": null,
"CapDrop": null,
"Dns": null,
"DnsOptions": null,
"DnsSearch": null,
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "container:0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 936,
"PidMode": "",
"Privileged": false,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": [
"seccomp=unconfined"
],
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"ConsoleSize": [
0,
0
],
"Isolation": "",
"CpuShares": 1024,
"Memory": 1476395008,
"NanoCpus": 0,
"CgroupParent": "kubepods-burstable-pod74a94609_b575_11e8_910e_0050568848e8.slice",
"BlkioWeight": 0,
"BlkioWeightDevice": null,
"BlkioDeviceReadBps": null,
"BlkioDeviceWriteBps": null,
"BlkioDeviceReadIOps": null,
"BlkioDeviceWriteIOps": null,
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DiskQuota": 0,
"KernelMemory": 0,
"MemoryReservation": 0,
"MemorySwap": 2952790016,
"MemorySwappiness": -1,
"OomKillDisable": false,
"PidsLimit": 0,
"Ulimits": null,
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0
},
"GraphDriver": {
"Name": "overlay",
"Data": {
"LowerDir": "/var/lib/docker/overlay/9e0164c7e2d049bbf3260f6a332b2995368d8af4d6f8357f8c70c18f8d4c7864/root",
"MergedDir": "/var/lib/docker/overlay/482521c521bccd54709755eb4ffb21fe2a0f63171ca2ee576f4477736cb2b25c/merged",
"UpperDir": "/var/lib/docker/overlay/482521c521bccd54709755eb4ffb21fe2a0f63171ca2ee576f4477736cb2b25c/upper",
"WorkDir": "/var/lib/docker/overlay/482521c521bccd54709755eb4ffb21fe2a0f63171ca2ee576f4477736cb2b25c/work"
}
},
"Mounts": [
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-jars-volume",
"Destination": "/var/spark-data/spark-jars",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/download-files",
"Destination": "/var/spark-data/spark-files",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~secret/default-token-jxg4k",
"Destination": "/var/run/secrets/kubernetes.io/serviceaccount",
"Mode": "ro",
"RW": false,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/etc-hosts",
"Destination": "/etc/hosts",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/containers/executor/664b5635",
"Destination": "/dev/termination-log",
"Mode": "",
"RW": true,
"Propagation": ""
},
{
"Type": "bind",
"Source": "/var/lib/kubelet/pods/74a94609-b575-11e8-910e-0050568848e8/volumes/kubernetes.io~empty-dir/spark-local-dir-0-spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58",
"Destination": "/mnt/tmp/spark-local/spark-a85d0ea9-fa9a-4ade-a3f2-45b94d72ec58",
"Mode": "",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "spark-job-on-k8s-100-1536637851433-exec-1",
"Domainname": "",
"User": "0",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"SPARK_EXECUTOR_ID=1",
"SPARK_EXECUTOR_POD_IP=161.161.76.219",
"SPARK_MOUNTED_FILES_DIR=/var/spark-data/spark-files",
"SPARK_EXECUTOR_PORT=10000",
"SPARK_DRIVER_URL=spark://CoarseGrainedScheduler@spark-job-on-k8s-100-1536637851433-driver-svc.spark-cluster.svc.dev161:7078",
"SPARK_EXECUTOR_CORES=1",
"SPARK_EXECUTOR_MEMORY=1024m",
"SPARK_APPLICATION_ID=spark-1619ed3f66b04bf9a54a7230a1f551a4",
"SPARK_MOUNTED_CLASSPATH=/var/spark-data/spark-jars/*",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT=tcp://161.161.52.23:4040",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.52.23",
"KUBERNETES_PORT_443_TCP_ADDR=161.161.0.1",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_PORT=4040",
"KUBERNETES_PORT_443_TCP=tcp://161.161.0.1:443",
"KUBERNETES_PORT_443_TCP_PROTO=tcp",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_HOST=161.161.13.152",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT_DRIVER_UI_PORT=4040",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_SERVICE_HOST=161.161.52.23",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PROTO=tcp",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP_PORT=4040",
"KUBERNETES_SERVICE_HOST=161.161.0.1",
"KUBERNETES_SERVICE_PORT_HTTPS=443",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_SERVICE_PORT=4040",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP_ADDR=161.161.13.152",
"COM_INSPUR_IOT_RULEENGINE_1533716701881_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.52.23:4040",
"KUBERNETES_SERVICE_PORT=443",
"KUBERNETES_PORT=tcp://161.161.0.1:443",
"KUBERNETES_PORT_443_TCP_PORT=443",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT=tcp://161.161.13.152:4040",
"COM_INSPUR_IOT_RULEENGINE_1534918928964_DRIVER_SVC_UI_PORT_4040_TCP=tcp://161.161.13.152:4040",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/lib/jvm/java-1.8-openjdk/jre/bin:/usr/lib/jvm/java-1.8-openjdk/bin",
"LANG=C.UTF-8",
"JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk",
"JAVA_VERSION=8u131",
"JAVA_ALPINE_VERSION=8.131.11-r2",
"SPARK_HOME=/opt/spark"
],
"Cmd": [
"/bin/sh",
"-c",
"SPARK_CLASSPATH=\"${SPARK_HOME}/jars/*\" && env | grep SPARK_JAVA_OPT_ | sed 's/[^=]*=\\(.*\\)/\\1/g' > /tmp/java_opts.txt && readarray -t SPARK_EXECUTOR_JAVA_OPTS < /tmp/java_opts.txt && if ! [ -z ${SPARK_MOUNTED_CLASSPATH}+x} ]; then SPARK_CLASSPATH=\"$SPARK_MOUNTED_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXECUTOR_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXECUTOR_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_EXTRA_CLASSPATH+x} ]; then SPARK_CLASSPATH=\"$SPARK_EXTRA_CLASSPATH:$SPARK_CLASSPATH\"; fi && if ! [ -z ${SPARK_MOUNTED_FILES_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_DIR/.\" .; fi && if ! [ -z ${SPARK_MOUNTED_FILES_FROM_SECRET_DIR+x} ]; then cp -R \"$SPARK_MOUNTED_FILES_FROM_SECRET_DIR/.\" .; fi && ${JAVA_HOME}/bin/java \"${SPARK_EXECUTOR_JAVA_OPTS[@]}\" -Dspark.executor.port=$SPARK_EXECUTOR_PORT -Xms$SPARK_EXECUTOR_MEMORY -Xmx$SPARK_EXECUTOR_MEMORY -cp $SPARK_CLASSPATH org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url $SPARK_DRIVER_URL --executor-id $SPARK_EXECUTOR_ID --cores $SPARK_EXECUTOR_CORES --app-id $SPARK_APPLICATION_ID --hostname $SPARK_EXECUTOR_POD_IP"
],
"Healthcheck": {
"Test": [
"NONE"
]
},
"ArgsEscaped": true,
"Image": "registry.dev161.com:5000/iot/spark-executor@sha256:c8650704517e254751fcef052eff13e3e4e40cff4b02039aceb9ce050f77d9a3",
"Volumes": null,
"WorkingDir": "/opt/spark/work-dir",
"Entrypoint": [
"/opt/entrypoint.sh"
],
"OnBuild": null,
"Labels": {
"annotation.io.kubernetes.container.hash": "3136f4c3",
"annotation.io.kubernetes.container.ports": "[{\"name\":\"executor\",\"containerPort\":10000,\"protocol\":\"TCP\"},{\"name\":\"blockmanager\",\"containerPort\":7079,\"protocol\":\"TCP\"}]",
"annotation.io.kubernetes.container.restartCount": "0",
"annotation.io.kubernetes.container.terminationMessagePath": "/dev/termination-log",
"annotation.io.kubernetes.container.terminationMessagePolicy": "File",
"annotation.io.kubernetes.pod.terminationGracePeriod": "30",
"io.kubernetes.container.logpath": "/var/log/pods/74a94609-b575-11e8-910e-0050568848e8/executor_0.log",
"io.kubernetes.container.name": "executor",
"io.kubernetes.docker.type": "container",
"io.kubernetes.pod.name": "spark-job-on-k8s-100-1536637851433-exec-1",
"io.kubernetes.pod.namespace": "spark-cluster",
"io.kubernetes.pod.uid": "74a94609-b575-11e8-910e-0050568848e8",
"io.kubernetes.sandbox.id": "0c5997eecdb5c81783f79e70be703a607762ba813dff722e58971899dc4bda1b"
}
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "",
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"Ports": null,
"SandboxKey": "",
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {}
}
}
]
resource-staging-server:
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: 2018-09-11T10:39:29Z
generateName: spark-resource-staging-server-544f9dfdc7-
labels:
pod-template-hash: "1009589873"
resource-staging-server-instance: default
name: spark-resource-staging-server-544f9dfdc7-6z9qm
namespace: iop
ownerReferences:
- apiVersion: extensions/v1beta1
blockOwnerDeletion: true
controller: true
kind: ReplicaSet
name: spark-resource-staging-server-544f9dfdc7
uid: f64cb936-b5ae-11e8-a973-6c92bf88f88e
resourceVersion: "5271010"
selfLink: /api/v1/namespaces/iop/pods/spark-resource-staging-server-544f9dfdc7-6z9qm
uid: f64ec3f8-b5ae-11e8-a973-6c92bf88f88e
spec:
containers:
- args:
- /etc/spark-resource-staging-server/resource-staging-server.properties
image: registry.inspurcloud.cn/service/bigdata/spark-resource-staging-server:v2.2.0-kubernetes-0.4.0
imagePullPolicy: IfNotPresent
name: spark-resource-staging-server
resources:
limits:
cpu: "1"
memory: 2Gi
requests:
cpu: 100m
memory: 256Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /etc/spark-resource-staging-server
name: resource-staging-server-properties
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: default-token-ztnhf
readOnly: true
dnsPolicy: ClusterFirst
imagePullSecrets:
- name: service-registry
nodeName: slave1
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
serviceAccount: default
serviceAccountName: default
terminationGracePeriodSeconds: 30
volumes:
- configMap:
defaultMode: 420
name: spark-resource-staging-server-config
name: resource-staging-server-properties
- name: default-token-ztnhf
secret:
defaultMode: 420
secretName: default-token-ztnhf
status:
conditions:
- lastProbeTime: null
lastTransitionTime: 2018-09-11T10:39:29Z
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: 2018-09-11T10:39:42Z
status: "True"
type: Ready
- lastProbeTime: null
lastTransitionTime: 2018-09-11T10:39:29Z
status: "True"
type: PodScheduled
containerStatuses:
- containerID: docker://433aec71282a7371c091c69f7f921a5612f2ac98ec26c758629836c4855ebb1f
image: registry.inspurcloud.cn/service/bigdata/spark-resource-staging-server:v2.2.0-kubernetes-0.4.0
imageID: docker-pullable://registry.inspurcloud.cn/service/bigdata/spark-resource-staging-server@sha256:9480a49fb7c1ff6d6da85dcdc034a2e0a0006fb3d3a424671bf174c055b538d3
lastState: {}
name: spark-resource-staging-server
ready: true
restartCount: 0
state:
running:
startedAt: 2018-09-11T10:39:42Z
hostIP: 10.100.0.17
phase: Running
podIP: 214.213.140.247
qosClass: Burstable
startTime: 2018-09-11T10:39:29Z免责声明：本站发布的内容（图片、视频和文字）以原创、转载和分享为主，文章观点不代表本网站立场，如果涉及侵权请联系站长邮箱：is@yisu.com进行举报，并提供相关证据，一经查实，将立刻删除涉嫌侵权内容。
