(consumer) --> (redis-ambassador) ---network---> (redis-ambassador) --> (redis)
官方提供的image,其中运行socat进行redis数据转发。
paas@ubuntu:~$ docker run -d --name redis shipyard/redis
ee76fff375499f5f69060633e3c07edcf92d264d41b6e71ad4123891bb9808ba
paas@ubuntu:~$ docker inspect 5ea7e8765ad4f94e97514abadf730a682a16fe6ed4ef221ea0b659e2381b256b
paas@ubuntu:~$ docker inspect ee76fff375499f5f69060633e3c07edcf92d264d41b6e71ad4123891bb9808ba
[{
"ID": "ee76fff375499f5f69060633e3c07edcf92d264d41b6e71ad4123891bb9808ba",
"Created": "2014-05-08T03:15:28.374074044Z",
"Path": "/usr/local/bin/redis-server",
"Args": [
"/etc/redis.conf"
],
"Config": {
"Hostname": "ee76fff37549",
"Domainname": "",
"User": "",
"Memory": 0,
"MemorySwap": 0,
"CpuShares": 0,
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"PortSpecs": null,
"ExposedPorts": {
"6379/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"HOME=/",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": [
"/etc/redis.conf"
],
"Image": "shipyard/redis",
"Volumes": {
"/var/lib/redis": {}
},
"WorkingDir": "",
"Entrypoint": [
"/usr/local/bin/redis-server"
],
"NetworkDisabled": false,
"OnBuild": null
},
"State": {
"Running": true,
"Pid": 3244,
"ExitCode": 0,
"StartedAt": "2014-05-08T03:16:16.392818522Z",
"FinishedAt": "2014-05-08T03:15:42.985608485Z",
"Ghost": false
},
"Image": "b48b681ac98459fdc7efc98c9cce7e25f8b1537a4496aad9d355b3b11ca7144a",
"NetworkSettings": {
"IPAddress": "172.17.0.2",
"IPPrefixLen": 16,
"Gateway": "172.17.42.1",
"Bridge": "docker0",
"PortMapping": null,
"Ports": {
"6379/tcp": null
}
},
"ResolvConfPath": "/var/lib/docker/containers/ee76fff375499f5f69060633e3c07edcf92d264d41b6e71ad4123891bb9808ba/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/ee76fff375499f5f69060633e3c07edcf92d264d41b6e71ad4123891bb9808ba/hostname",
"HostsPath": "/var/lib/docker/containers/ee76fff375499f5f69060633e3c07edcf92d264d41b6e71ad4123891bb9808ba/hosts",
"Name": "/redis",
"Driver": "aufs",
"ExecDriver": "native-0.1",
"Volumes": {
"/var/lib/redis": "/var/lib/docker/vfs/dir/f8a140fcfba06f9fc454d30a11a23c0ed981b712a8b29f913fdbbd4f2d5a9653"
},
"VolumesRW": {
"/var/lib/redis": true
},
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LxcConf": [],
"Privileged": false,
"PortBindings": {
"6379/tcp": null
},
"Links": null,
"PublishAllPorts": false,
"Dns": null,
"DnsSearch": null,
"VolumesFrom": null
}
}]
paas@ubuntu:~$ sudo iptables -S
-P INPUT ACCEPT
-P FORWARD ACCEPT
-P OUTPUT ACCEPT
-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -i docker0 ! -o docker0 -j ACCEPT
-A FORWARD -i docker0 -o docker0 -j ACCEPT
paas@ubuntu:~$ sudo iptables -S -t nat
-P PREROUTING ACCEPT
-P INPUT ACCEPT
-P OUTPUT ACCEPT
-P POSTROUTING ACCEPT
-N DOCKER
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-A POSTROUTING -s 172.17.0.0/16 ! -d 172.17.0.0/16 -j MASQUERADE
可以看到,此时redis-server只是运行在容器中。系统中没有配置任何转发规则,外部无法访问到redis-server的服务。
paas@ubuntu:~$ docker run -d --link redis:redis --name redis_ambassador -p 6379:6379 svendowideit/ambassador
4480d8897fdedc1f34f0460d3853e4cc696a3751891f5783ebc12c44d985e68e
paas@ubuntu:~$ docker inspect 4480d8897fdedc1f34f0460d3853e4cc696a3751891f5783ebc12c44d985e68e
[{
"ID": "4480d8897fdedc1f34f0460d3853e4cc696a3751891f5783ebc12c44d985e68e",
"Created": "2014-05-08T03:19:28.369725238Z",
"Path": "/bin/sh",
"Args": [
"-c",
"env | grep _TCP= | sed 's/.*_PORT_\\([0-9]*\\)_TCP=tcp:\\/\\/\\(.*\\):\\(.*\\)/socat TCP4-LISTEN:\\1,fork,reuseaddr TCP4:\\2:\\3 \\\u0026/' | sh \u0026\u0026 top"
],
"Config": {
"Hostname": "4480d8897fde",
"Domainname": "",
"User": "",
"Memory": 0,
"MemorySwap": 0,
"CpuShares": 0,
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"PortSpecs": null,
"ExposedPorts": {
"6379/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"HOME=/",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": [
"/bin/sh",
"-c",
"env | grep _TCP= | sed 's/.*_PORT_\\([0-9]*\\)_TCP=tcp:\\/\\/\\(.*\\):\\(.*\\)/socat TCP4-LISTEN:\\1,fork,reuseaddr TCP4:\\2:\\3 \\\u0026/' | sh \u0026\u0026 top"
],
"Image": "svendowideit/ambassador",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": null,
"NetworkDisabled": false,
"OnBuild": null
},
"State": {
"Running": true,
"Pid": 3316,
"ExitCode": 0,
"StartedAt": "2014-05-08T03:19:28.466496993Z",
"FinishedAt": "0001-01-01T00:00:00Z",
"Ghost": false
},
"Image": "0d2200edc53e6e8a201d7d42ddcac3235bfb8655223ba1029d3e7765736f2a75",
"NetworkSettings": {
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"Gateway": "172.17.42.1",
"Bridge": "docker0",
"PortMapping": null,
"Ports": {
"6379/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "6379"
}
]
}
},
"ResolvConfPath": "/var/lib/docker/containers/4480d8897fdedc1f34f0460d3853e4cc696a3751891f5783ebc12c44d985e68e/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/4480d8897fdedc1f34f0460d3853e4cc696a3751891f5783ebc12c44d985e68e/hostname",
"HostsPath": "/var/lib/docker/containers/4480d8897fdedc1f34f0460d3853e4cc696a3751891f5783ebc12c44d985e68e/hosts",
"Name": "/redis_ambassador",
"Driver": "aufs",
"ExecDriver": "native-0.1",
"Volumes": {},
"VolumesRW": {},
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LxcConf": [],
"Privileged": false,
"PortBindings": {
"6379/tcp": [
{
"HostIp": "0.0.0.0",
"HostPort": "6379"
}
]
},
"Links": null,
"PublishAllPorts": false,
"Dns": null,
"DnsSearch": null,
"VolumesFrom": null
}
}]
paas@ubuntu:~$ sudo iptables -S
-P INPUT ACCEPT
-P FORWARD ACCEPT
-P OUTPUT ACCEPT
-A FORWARD -s 172.17.0.2/32 -d 172.17.0.3/32 -i docker0 -o docker0 -p tcp -m tcp --sport 6379 -j ACCEPT
-A FORWARD -s 172.17.0.3/32 -d 172.17.0.2/32 -i docker0 -o docker0 -p tcp -m tcp --dport 6379 -j ACCEPT
-A FORWARD -d 172.17.0.3/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 6379 -j ACCEPT
-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -i docker0 ! -o docker0 -j ACCEPT
-A FORWARD -i docker0 -o docker0 -j ACCEPT
paas@ubuntu:~$ sudo iptables -S -t nat
-P PREROUTING ACCEPT
-P INPUT ACCEPT
-P OUTPUT ACCEPT
-P POSTROUTING ACCEPT
-N DOCKER
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-A POSTROUTING -s 172.17.0.0/16 ! -d 172.17.0.0/16 -j MASQUERADE
-A DOCKER -p tcp -m tcp --dport 6379 -j DNAT --to-destination 172.17.0.3:6379
paas@ubuntu:~$ ps -ef | grep socat
root 3316 367 0 11:19 ? 00:00:00 /bin/sh -c env | grep _TCP= | sed 's/.*_PORT_\([0-9]*\)_TCP=tcp:\/\/\(.*\):\(.*\)/socat TCP4-LISTEN:\1,fork,reuseaddr TCP4:\2:\3 \&/' | sh && top
root 3350 3316 0 11:19 ? 00:00:00 socat TCP4-LISTEN:6379,fork,reuseaddr TCP4:172.17.0.2:6379
paas 3376 2952 0 11:23 pts/4 00:00:00 grep --color=auto socat
paas@ubuntu:~$ sudo netstat -nalp | grep 6379
tcp6 0 0 :::6379 :::* LISTEN 367/docker
docker run -d --link redis:redis --name redis_ambassador -p 6379:6379 svendowideit/ambassador中的-p 6379:6379导致增加了以下iptables规则
-A FORWARD -d 172.17.0.3/32 ! -i docker0 -o docker0 -p tcp -m tcp --dport 6379 -j ACCEPT
-A DOCKER -p tcp -m tcp --dport 6379 -j DNAT --to-destination 172.17.0.3:6379
本机的docker进程监听6379端口。
将访问本机6379端口的请求转发到172.17.0.3:6379,即socat进程(3350)。
docker run -d --link redis:redis --name redis_ambassador -p 6379:6379 svendowideit/ambassador中的--link redis:redis导致增加了以下iptables规则
-A FORWARD -s 172.17.0.2/32 -d 172.17.0.3/32 -i docker0 -o docker0 -p tcp -m tcp --sport 6379 -j ACCEPT
-A FORWARD -s 172.17.0.3/32 -d 172.17.0.2/32 -i docker0 -o docker0 -p tcp -m tcp --dport 6379 -j ACCEPT
socat TCP4-LISTEN:6379,fork,reuseaddr TCP4:172.17.0.2:6379
表示监听6379端口,从这个端口上收到的数据转发到172.17.0.2:6379
paas@ubuntu:~$ sudo docker run -d --name redis_ambassador --expose 6379 -e REDIS_PORT_6379_TCP=tcp://192.168.0.71:6379 svendowideit/ambassador
507652fd48d308ad2f7d3a206dacd899776c2458eb103112843b0419d175a75a
paas@ubuntu:~$ sudo docker inspect 507652fd48d308ad2f7d3a206dacd899776c2458eb103112843b0419d175a75a
[{
"ID": "507652fd48d308ad2f7d3a206dacd899776c2458eb103112843b0419d175a75a",
"Created": "2014-05-08T05:00:21.137393924Z",
"Path": "/bin/sh",
"Args": [
"-c",
"env | grep _TCP= | sed 's/.*_PORT_\\([0-9]*\\)_TCP=tcp:\\/\\/\\(.*\\):\\(.*\\)/socat TCP4-LISTEN:\\1,fork,reuseaddr TCP4:\\2:\\3 \\\u0026/' | sh \u0026\u0026 top"
],
"Config": {
"Hostname": "507652fd48d3",
"Domainname": "",
"User": "",
"Memory": 0,
"MemorySwap": 0,
"CpuShares": 0,
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"PortSpecs": null,
"ExposedPorts": {
"6379/tcp": {}
},
"Tty": false,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"REDIS_PORT_6379_TCP=tcp://192.168.0.71:6379",
"HOME=/",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": [
"/bin/sh",
"-c",
"env | grep _TCP= | sed 's/.*_PORT_\\([0-9]*\\)_TCP=tcp:\\/\\/\\(.*\\):\\(.*\\)/socat TCP4-LISTEN:\\1,fork,reuseaddr TCP4:\\2:\\3 \\\u0026/' | sh \u0026\u0026 top"
],
"Image": "svendowideit/ambassador",
"Volumes": null,
"WorkingDir": "",
"Entrypoint": null,
"NetworkDisabled": false,
"OnBuild": null
},
"State": {
"Running": true,
"Pid": 17124,
"ExitCode": 0,
"StartedAt": "2014-05-08T05:00:21.172505186Z",
"FinishedAt": "0001-01-01T00:00:00Z",
"Ghost": false
},
"Image": "0d2200edc53e6e8a201d7d42ddcac3235bfb8655223ba1029d3e7765736f2a75",
"NetworkSettings": {
"IPAddress": "172.17.0.2",
"IPPrefixLen": 16,
"Gateway": "172.17.42.1",
"Bridge": "docker0",
"PortMapping": null,
"Ports": {
"6379/tcp": null
}
},
"ResolvConfPath": "/etc/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/507652fd48d308ad2f7d3a206dacd899776c2458eb103112843b0419d175a75a/hostname",
"HostsPath": "/var/lib/docker/containers/507652fd48d308ad2f7d3a206dacd899776c2458eb103112843b0419d175a75a/hosts",
"Name": "/redis_ambassador",
"Driver": "aufs",
"ExecDriver": "native-0.1",
"Volumes": {},
"VolumesRW": {},
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LxcConf": [],
"Privileged": false,
"PortBindings": {
"6379/tcp": null
},
"Links": null,
"PublishAllPorts": false,
"Dns": null,
"DnsSearch": null,
"VolumesFrom": null
}
}]
paas@ubuntu:~$ sudo iptables -S
-P INPUT ACCEPT
-P FORWARD ACCEPT
-P OUTPUT ACCEPT
-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -i docker0 ! -o docker0 -j ACCEPT
-A FORWARD -i docker0 -o docker0 -j ACCEPT
paas@ubuntu:~$ sudo iptables -S -t nat
-P PREROUTING ACCEPT
-P INPUT ACCEPT
-P OUTPUT ACCEPT
-P POSTROUTING ACCEPT
-N DOCKER
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-A POSTROUTING -s 172.17.0.0/16 ! -d 172.17.0.0/16 -j MASQUERADE
paas@ubuntu:~$ ps -ef | grep socat
root 17124 1789 0 13:00 ? 00:00:00 /bin/sh -c env | grep _TCP= | sed 's/.*_PORT_\([0-9]*\)_TCP=tcp:\/\/\(.*\):\(.*\)/socat TCP4-LISTEN:\1,fork,reuseaddr TCP4:\2:\3 \&/' | sh && top
root 17146 17124 0 13:00 ? 00:00:00 socat TCP4-LISTEN:6379,fork,reuseaddr TCP4:192.168.0.71:6379
paas 17164 16719 0 13:03 pts/1 00:00:00 grep --color=auto socat
进程环境变量
sudo cat /proc/17146/environ
HOSTNAME=507652fd48d3
HOME=/
REDIS_PORT_6379_TCP=tcp://192.168.0.71:6379
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
PWD=/
socat TCP4-LISTEN:6379,fork,reuseaddr TCP4:192.168.0.71:6379 socat监听6379端口,将收到的数据转发到192.168.0.71:6379。是在容器中监听,在本机上是看不到有进程监听6379端口的,需要后续的iptables的nat规则配合,socat才能收到数据。
sudo docker run -d --name redis_ambassador --expose 6379 -e REDIS_PORT_6379_TCP=tcp://192.168.0.71:6379 svendowideit/ambassador中,--expose 6379表示允许link到此容器的容器访问6379端口,-e REDIS_PORT_6379_TCP=tcp://192.168.0.71:6379为设置访问redis的环境变量。
REDIS_PORT_6379_TCP这个环境变量的格式是固定的,socat命令需要从中获取监听端口,如果改成例如REDIS_PORT_1234_TCP,那么此容器中的socat命令将会变成socat TCP4-LISTEN:1234,fork,reuseaddr TCP4:192.168.0.71:6379
paas@ubuntu:~$ sudo docker run -i -t --rm --link redis_ambassador:redis relateiq/redis-cli
redis 172.17.0.2:6379>
paas@ubuntu:~$ sudo docker inspect 043ab6f80052
[{
"ID": "043ab6f80052b14fad50ec4d01f3b550b4b8a1aa053394dc314c86b50443b6e3",
"Created": "2014-05-08T05:48:31.872337998Z",
"Path": "/start.sh",
"Args": [],
"Config": {
"Hostname": "043ab6f80052",
"Domainname": "",
"User": "",
"Memory": 0,
"MemorySwap": 0,
"CpuShares": 0,
"AttachStdin": true,
"AttachStdout": true,
"AttachStderr": true,
"PortSpecs": null,
"ExposedPorts": {},
"Tty": true,
"OpenStdin": true,
"StdinOnce": true,
"Env": [
"HOME=/",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": null,
"Image": "relateiq/redis-cli",
"Volumes": {},
"WorkingDir": "",
"Entrypoint": [
"/start.sh"
],
"NetworkDisabled": false,
"OnBuild": null
},
"State": {
"Running": true,
"Pid": 17373,
"ExitCode": 0,
"StartedAt": "2014-05-08T05:48:31.965979461Z",
"FinishedAt": "0001-01-01T00:00:00Z",
"Ghost": false
},
"Image": "616b2d3dc69c2b655e2a0b98dca40bbc629c5c812eb5b62afcd8311dace6feb0",
"NetworkSettings": {
"IPAddress": "172.17.0.3",
"IPPrefixLen": 16,
"Gateway": "172.17.42.1",
"Bridge": "docker0",
"PortMapping": null,
"Ports": {}
},
"ResolvConfPath": "/etc/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/043ab6f80052b14fad50ec4d01f3b550b4b8a1aa053394dc314c86b50443b6e3/hostname",
"HostsPath": "/var/lib/docker/containers/043ab6f80052b14fad50ec4d01f3b550b4b8a1aa053394dc314c86b50443b6e3/hosts",
"Name": "/mad_hoover",
"Driver": "aufs",
"ExecDriver": "native-0.1",
"Volumes": {},
"VolumesRW": {},
"HostConfig": {
"Binds": null,
"ContainerIDFile": "",
"LxcConf": [],
"Privileged": false,
"PortBindings": {},
"Links": null,
"PublishAllPorts": false,
"Dns": null,
"DnsSearch": null,
"VolumesFrom": null
}
}]
可以看到这个容器中执行的是start.sh。start.sh内容如下
#!/bin/bash
if [ $# -eq 0 ]; then
/redis/src/redis-cli -h $REDIS_PORT_6379_TCP_ADDR -p $REDIS_PORT_6379_TCP_PORT
else
/redis/src/redis-cli "$@"
fi
start.sh进程的环境变量
sudo cat /proc/17373/environ
HOME=/
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
HOSTNAME=043ab6f80052
TERM=xterm
REDIS_PORT=tcp://172.17.0.2:6379
REDIS_PORT_6379_TCP=tcp://172.17.0.2:6379
REDIS_PORT_6379_TCP_ADDR=172.17.0.2
REDIS_PORT_6379_TCP_PORT=6379
REDIS_PORT_6379_TCP_PROTO=tcp
REDIS_NAME=/mad_hoover/redis
REDIS_ENV_REDIS_PORT_6379_TCP=tcp://192.168.0.71:6379
paas@ubuntu:~$ sudo iptables -S
-P INPUT ACCEPT
-P FORWARD ACCEPT
-P OUTPUT ACCEPT
-A FORWARD -s 172.17.0.2/32 -d 172.17.0.3/32 -i docker0 -o docker0 -p tcp -m tcp --sport 6379 -j ACCEPT
-A FORWARD -s 172.17.0.3/32 -d 172.17.0.2/32 -i docker0 -o docker0 -p tcp -m tcp --dport 6379 -j ACCEPT
-A FORWARD -o docker0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
-A FORWARD -i docker0 ! -o docker0 -j ACCEPT
-A FORWARD -i docker0 -o docker0 -j ACCEPT
paas@ubuntu:~$ sudo iptables -S -t nat
-P PREROUTING ACCEPT
-P INPUT ACCEPT
-P OUTPUT ACCEPT
-P POSTROUTING ACCEPT
-N DOCKER
-A PREROUTING -m addrtype --dst-type LOCAL -j DOCKER
-A OUTPUT ! -d 127.0.0.0/8 -m addrtype --dst-type LOCAL -j DOCKER
-A POSTROUTING -s 172.17.0.0/16 ! -d 172.17.0.0/16 -j MASQUERADE
以下环境变量的存在是sudo docker run -i -t --rm --link redis_ambassador:redis relateiq/redis-cli命令中的--link redis_ambassador:redis部分与sudo docker run -d --name redis_ambassador --expose 6379 -e REDIS_PORT_6379_TCP=tcp://192.168.0.71:6379 svendowideit/ambassador命令中的--expose 6379部分功能导致的,本质上是因为link存在expose的端口。详情可以参考docker之容器连接实现中关于links的代码。如果在sudo docker run -d --name redis_ambassador --expose 6379 -e REDIS_PORT_6379_TCP=tcp://192.168.0.71:6379 svendowideit/ambassador命令中去掉--expose 6379,则redis-cli进程会因为没有相应的环境变量而无法连接服务器。
REDIS_PORT=tcp://172.17.0.2:6379
REDIS_PORT_6379_TCP=tcp://172.17.0.2:6379
REDIS_PORT_6379_TCP_ADDR=172.17.0.2
REDIS_PORT_6379_TCP_PORT=6379
REDIS_PORT_6379_TCP_PROTO=tcp
从上面的过程跟踪可以了解以redis为例的docker容克跨主机连接实现方式,如果要解决其他服务的跨主机连接可以仿照此方式实现。主要是制作相应的ambassador容器(使用socat),确定环境变量规则,以及编写简单的客户端脚本。