ARCHIVE / 2019-08-12

websocket 達到百萬連接?

Published 2019-08-12

Programming & Software Engineering Backend & Distributed Systems

在跟技術經理討論過一次，我們的推送 server，將會面臨到有沒有可能斷線

或者資料遺失的可能性，

假設一最多只能有 65536 個port 扣掉連接埠後，剩下

可用連接埠可能兩三萬，那麼有沒有可能單台 server 就突破 65536 個 port呢 ?

emmmm…

在經過一番查詢是有的，那麼就是差實做囉，我們來架設一波

首先呢我們要對 linux 進行一些額外的設定

至於伺服器方面呢，我們把移到golang 的 websocket 伺服器 client 和 server 是run 在同一台虛擬機

client 端

docker搭建

https://joshhu.gitbooks.io/dockercommands/content/Containers/DockerRun.html

產生 n個 docker 映像檔並執行 client.go 編譯出來的 client 使用 docker 映射到 docker 容器裡再讓容器發起 m個連接到指定 ip 上

code

#!/bin/bash
## This script helps regenerating multiple client instances in different network namespaces using Docker
## This helps to overcome the ephemeral source port limitation
## Usage: ./connect <connections> <number of clients> <server ip>
## Server IP is usually the Docker gateway IP address, which is 172.17.0.1 by default
## Number of clients helps to speed up connections establishment at large scale, in order to make the demo faster

CONNECTIONS=$1
REPLICAS=$2
IP=$3
#go build --tags "static netgo" -o client client.go
for (( c=0; c<${REPLICAS}; c++ ))
do
    docker run -v $(pwd)/client:/client --name 1mclient_$c -d  alpine /client -conn=${CONNECTIONS} -ip=${IP}
done

client 發起 10萬連接

code

sudo bash setup.sh 20000 5 192.168.109.129

刪除迴圈產生出來的 docker 映像檔

code

sudo docker rm -f 1mclient_4
sudo docker rm -f 1mclient_3
sudo docker rm -f 1mclient_2
sudo docker rm -f 1mclient_1
sudo docker rm -f 1mclient_0

server 端

linux 調效篇

這篇講得不錯很容易懂

http://blog.udn.com/q928856957/25855941 https://blog.csdn.net/xiexingshishu/article/details/43373297 https://serverfault.com/questions/769978/sysctl-cannot-stat-proc-sys-net-ipv4-netfilter-ip-conntrack-max-no-such-file

code

sudo sysctl -w fs.file-max=2000500
sudo sysctl -w fs.nr_open=2000500
sudo sysctl -w net.nf_conntrack_max=2000500
sudo ulimit -n 2000500
sudo sysctl -w net.ipv4.tcp_mem='131072  262144  524288'
sudo sysctl -w net.ipv4.tcp_rmem='8760  256960  4088000'
sudo sysctl -w net.ipv4.tcp_wmem='8760  256960  4088000'
sudo sysctl -w net.core.rmem_max=16384
sudo sysctl -w net.core.wmem_max=16384
sudo sysctl -w net.core.somaxconn=2048
sudo sysctl -w net.ipv4.tcp_max_syn_backlog=2048
sudo sysctl -w /proc/sys/net/core/netdev_max_backlog=2048
sudo sysctl -w net.ipv4.tcp_tw_recycle=1
sudo sysctl -w net.ipv4.tcp_tw_reuse=1
sudo echo 2000500 > /proc/sys/fs/nr_open

https://www.google.com/search?q=centos+開啟+端口&rlz=1C1WPZA_enTW818TW818&oq=centos+開啟+端口&aqs=chrome..69i57.6505j0j7&sourceid=chrome&ie=UTF-8

Run server!

組態檔，我們把onlyTestConnect 設為 false

code

{
        "port": 8888,
        "delay": "1m",
        "interval": "1m",
        "totalSize": 1,
        "onlyTestConnect": false
}

server.go

code

package main

import (
 "container/list"
 "fmt"
 "golang.org/x/net/websocket"
 "net/http"
 "strconv"
 "time"
 "encoding/json"
 "os"
 "sync"
 "goserver"
 "math/rand"
 //"unsafe"
)

var Config goserver.Configuration
const n = 10
var wsList = [n]*list.List {list.New(),list.New(),list.New(),list.New(),list.New(),list.New(),list.New(),list.New(),list.New(),list.New()}
var locks = [n]sync.RWMutex {}

func wsHandler(ws *websocket.Conn) {
 //p := unsafe.Pointer(&ws)
 //index := ((int)(uintptr(p))) % n
 index := rand.Intn(n)
 lock := locks[index]
 lock.Lock()
 wsList[index].PushBack(ws)  
 lock.Unlock()

 for {
  var reply string
  if err := websocket.Message.Receive(ws, &reply); err != nil {
   fmt.Println("Can't receive because of " + err.Error())
   break
  }
 }

 lock.Lock()
 for e := wsList[index].Front(); e != nil; e = e.Next() {
  if e.Value.(*websocket.Conn) == ws {
   wsList[index].Remove(e)
   break
  }
 }
 lock.Unlock()
}

func load(configfile string) goserver.Configuration {
 config := goserver.Configuration{}
 file, _ := os.Open(configfile)
 decoder := json.NewDecoder(file)
 err := decoder.Decode(&config)
 if err != nil {
  panic(err.Error())
 }

 return config
}

func main() {
 seed := time.Now().UTC().UnixNano()
 rand.Seed(seed)

 Config = load("config.json")

 //http.Handle("/", websocket.Handler(wsHandler))
 http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
  s := websocket.Server{Handler: websocket.Handler(wsHandler)}
  s.ServeHTTP(w, req)
 })

 delay, _ := time.ParseDuration(Config.Delay)
 interval, _ := time.ParseDuration(Config.Interval)
 timer := time.NewTimer(delay)
 if !Config.OnlyTestConnect {
  go func() {
   for {
    <-timer.C
    timer.Reset(interval)
    totalLen := 0
    for i :=0; i < n; i++ {
     totalLen += wsList[i].Len()
    }
    if totalLen >= Config.TotalSize {
     fmt.Println("send timestamp to all")
     for i :=0; i < n; i++ {
      i := i
      go func() {
       for e := wsList[i].Front(); e != nil; e = e.Next() {
        var ws = e.Value.(*websocket.Conn)
        now := time.Now().UnixNano() / int64(time.Millisecond)
        err := websocket.Message.Send(ws, strconv.FormatInt(now, 10))
        if err != nil {
         panic("Error: " + err.Error())
        }
       }
      }()

     }     
    } else {
     fmt.Println("current websockets: " + strconv.Itoa(totalLen))
    }

   }
  }()
 }

 err := http.ListenAndServe(":"+strconv.Itoa(Config.Port), nil)
 if err != nil {
  panic("Error: " + err.Error())
 }
}

emm… emmemm. 看起來差不多調校好了?? 記憶體吃滿了先開 10w client 玩玩看，不過單台確實突破 65536 的迷思 (?

中間能不能再夾一層 nginx 呢進行更多變化呢或優化呢… 看來要在買一些洋垃圾組一台 server

連接埠重複使用? 假象?

https://colobu.com/2019/02/23/1m-go-tcp-connection/
在看了下數據後最下面貼文發現了是不是只是本地端的假象? 來實驗一下
在使用docker 虛擬連接發現docker 創建的連線數是正常的嗎
開了兩台 client 模擬客戶端每台上限65536，可以看到 server 確實達到
接近12萬連線數所以這個就取決的server的硬件是否強大囉