consul-agent

字数 1466阅读 161

consul agent 分client server

client 一般部署在靠近应用的地方,甚至本机用于对应用的读写等(本主题只讨论读写)做出转发

server 参加raft选举,只有leader server负责写,一般也负责读(除非开启一致性模式的stale)

入口 github.com/hashicorp/consul/main.go

入口

func main() {

os.exit(realMain())

}

真实入口

func realMain() int {

...

cmds := command.Map(ui)

cli := &cli.CLI{

Args:        args, 

Commands:    cmds,

Autocomplete: true,

Name:        "consul",

HelpFunc:    cli.FilteredHelpFunc(names, cli.BasicHelpFunc("consul")),

}

exitCode, err := cli.Run()

...

}


命令加载

func Map(ui cli.Ui) map[string]cli.CommandFactory {

m := make(map[string]cli.CommandFactory)

for name, fn := range registry {

thisFn := fn

m[name] = func() (cli.Command, error) {

  return thisFn(ui)

}

}

return m

}

初始化时候加载命令

func init() {

...

Register("agent", func(ui cli.Ui) (cli.Command, error) {return agent.New(ui, rev, ver, verPre, verHuman, make(chan struct{})), nil...    })

agent 命令运行

func (c *cmd) Run(args []string) int {

...

code := c.run(args)

... 

}

agent 运行

func (c *cmd) run(args []string) int {

...

agent, err := agent.New(config)

...

if err := agent.Start(); err != nil {

...


}


agent 启动

func (a *Agent) Start() error {

...

if c.ServerMode {

...

server, err := consul.NewServerLogger(consulCfg,a.logger,a.tokens)

...

a.delegate = server

...

}else {

...

client, err := consul.NewClientLogger(consulCfg,a.logger)

...

a.delegate = client

...

servers, err := a.listenHTTP()

...

for _, srv := range servers {

if err := a.serveHTTP(srv); err != nil {

return err


}


}

...


}


agent http批量启动


func (a *Agent) serveHTTP(srv *HTTPServer) error {

...

err := srv.Serve(srv.ln)

...

}


agent 初始化http服务


func (a *Agent) listenHTTP() ([]*HTTPServer, error) {

...

srv.Server.Handler = srv.handler(a.config.EnableDebug)

...


}


注册agent 路由


func (s *HTTPServer) handler(enableDebug bool) http.Handler {

...

for pattern, fn := range endpoints {

...

handleFuncMetrics(pattern, s.wrap(bound, methods))

...

...

}


聚合agent所需路由

func init() {

...

[registerEndpoint("/v1/kv/", []string{"GET", "PUT","DELETE"}, (*HTTPServer).KVSEndpoint)]

...   

}


agent 注册endpoints方法

func registerEndpoint(pattern string, methods []string, fn unboundEndpoint) {

...

endpoints[pattern] = fn

...

}


agent kv 入口

func (s *HTTPServer) KVSEndpoint(resp http.ResponseWriter, req *http.Request) (interface{}, error) {

...

switch req.Method {

case "GET":

if keyList {

return s.KVSGetKeys(resp, req, &args)


}

return s.KVSGet(resp, req, &args)

case "PUT":

return s.KVSPut(resp, req, &args)

case "DELETE":

return s.KVSDelete(resp, req, &args)

default:

return nil, MethodNotAllowedError{req.Method,[]string{"GET", "PUT", "DELETE"}}


}


...

}

agent 真实 kv get


func (s *HTTPServer) KVSGet(resp http.ResponseWriter, req *http.Request, args *structs.KeyRequest) (interface{}, error) {

...

if err := s.agent.RPC(method, &args, &out); err != nil {

return nil, err

}

...


}


agent发起rpc


func (a *Agent) RPC(method string, args interface{}, reply interface{}) error {

    ...


    return a.delegate.RPC(method, args, reply)


    ...


}


client初始化


func NewClientLogger(config *Config, logger *log.Logger) (*Client, error) {

...

...


}


agent代理(client)发起rpc


func (c *Client) RPC(method string, args interface{}, reply interface{}) error {

...

rpcErr := c.connPool.RPC(c.config.Datacenter,server.Addr,server.Version,method,server.UseTLS,args,reply)

...


}


client 发起rpc


func (p *ConnPool) RPC(dc string, addr net.Addr, version int, method string, useTLS bool, args interface{}, reply interface{}) error    {

...

err = msgpackrpc.CallWithCodec(sc.codec,method,args,reply)

...


}


server 初始化


func NewServerLogger(config *Config, logger *log.Logger, tokens *token.Store) (*Server, error) {

...

if err := s.setupRPC(tlsWrap); err != nil {

...

go s.listen(s.Listener)

...

}

...

if err := s.setupRaft(); err != nil {

s.Shutdown()

return nil, fmt.Errorf("Failed to start Raft: %v", err)


}

...

}


初始化rpc服务对应的endpoints


func init(){

...

registerEndpoint(func(s *Server) interface{} { return &KVS{s} })

...

}


server 初始化rpc


func (s *Server) setupRPC(tlsWrap tlsutil.DCWrapper) error {

...

ln, err := net.ListenTCP("tcp", s.config.RPCAddr)

s.Listener = ln

...

}


server 启动rpc服务


func (s *Server) listen(listener net.Listener) {

...

  go s.handleConn(conn, false)

...

}


server  处理 rpc


func (s *Server) handleConn(conn net.Conn, isTLS bool) {

...

typ := pool.RPCType(buf[0])

...

switch typ {

case pool.RPCConsul:

s.handleConsulConn(conn)

...


}



server 处理 consulconn


func (s *Server) handleConsulConn(conn net.Conn) {

...

if err := s.rpcServer.ServeRequest(rpcCodec); err != nil {

...


}

...


}


server  kv get


func (k *KVS) Get(args *structs.KeyRequest, reply *structs.IndexedDirEntries) error {

...


如果是leader则返回false,不是则rpc获取

if done, err := k.srv.forward("KVS.Get", args,args,reply); done {

...


}

...

return k.srv.blockingQuery(

&args.QueryOptions,

&reply.QueryMeta,

func(ws memdb.WatchSet, state *state.Store) error {

index, ent, err := state.KVSGet(ws, args.Key)

...

if ent == nil {

if index == 0 {

reply.Index = 1

} else {

reply.Index = index

}

reply.Entries = nil

} else {

reply.Index = ent.ModifyIndex

reply.Entries = structs.DirEntries{ent}

}

return nil

...


}


client kv get 接口


func (s *Server) blockingQuery(queryOpts *structs.QueryOptions, queryMeta *structs.QueryMeta,fn queryFn) error {

...

开启一致性读

if queryOpts.RequireConsistent {

if err := s.consistentRead(); err != nil {

return err


}

...

err := fn(ws, state)

...


}


client kv put 接口


func (s *HTTPServer) KVSPut(resp http.ResponseWriter, req *http.Request, args *structs.KeyRequest) (interface{}, error) {

...

if err := s.agent.RPC("KVS.Apply", &applyReq, &out); err != nil {

    ...

}

...

}


rpc kv put 接口


func (k *KVS) Apply(args *structs.KVSRequest, reply *bool) error {

...

非leader则转发

if done, err := k.srv.forward("KVS.Apply", args, args, reply); done {

return err


}

...

resp, err := k.srv.raftApply(structs.KVSRequestType,args)

...


}


raft apply


func (s *Server) raftApply(t structs.MessageType, msg interface{}) (interface{}, error) {

...

future := s.raft.Apply(buf, enqueueLimit)

if err := future.Error(); err != nil {

return nil, err

}

return future.Response(), nil

...


}


raft apply真实入口


func (r *Raft) Apply(cmd []byte, timeout time.Duration) ApplyFuture {

...

logFuture := &logFuture{

log: Log{

Type: LogCommand,

Data: cmd,

},


}

logFuture.init()

select {

case <-timer:

return errorFuture{ErrEnqueueTimeout}

case <-r.shutdownCh:

return errorFuture{ErrRaftShutdown}

case r.applyCh <- logFuture:

return logFuture



}

...


}


leaderloop中apply此log


func (r *Raft) leaderLoop() {

...

case <-r.leaderState.commitCh:

...

r.processLogs(idx, commitLog)

...

case newLog := <-r.applyCh:

ready := []*logFuture{newLog}

批量

for i := 0; i < r.conf.MaxAppendEntries; i++ {

select {

case <-r.leaderState.commitCh:

case newLog := <-r.applyCh:

ready = append(ready, newLog)

default:break} }

...

r.dispatchLogs(ready)

...

}


存储日志,通知日志复制

func (r *Raft) dispatchLogs(applyLogs []*logFuture) {

...

r.leaderState.inflight.PushBack(applyLog)

...

if err := r.logs.StoreLogs(logs); err != nil {

...


}

...

for _, f := range r.leaderState.replState {

asyncNotifyCh(f.triggerCh)

}

...

...


}


复制循环


func (r *Raft) replicate(s *followerReplication) {


...


for !shouldStop {

select {

case <-s.triggerCh:

lastLogIdx, _ := r.getLastLog()

shouldStop = r.replicateTo(s, lastLogIdx)

}

}

... 

}


启动复制流程

func (r *Raft) replicateTo(s *followerReplication, lastIndex uint64) (shouldStop bool) {

...

if err := r.trans.AppendEntries(s.peer.ID, s.peer.Address, &req, &resp); err != nil {

...


}


if resp.Success {

updateLastAppended(s,&req)

...

}

...}



刚添加日志更新状态

func updateLastAppended(s *followerReplication, req *AppendEntriesRequest) {

...

if logs := req.Entries; len(logs) > 0 {

last := logs[len(logs)-1]

s.nextIndex = last.Index + 1

s.commitment.match(s.peer.ID, last.Index)

}

...

}


复制检查匹配日志索引


func (c *commitment) match(server ServerID, matchIndex uint64) {

c.Lock()

defer c.Unlock()

if prev, hasVote := c.matchIndexes[server]; hasVote && matchIndex > prev {

c.matchIndexes[server] = matchIndex

c.recalculate()


}


}



检查是否已经达到多数派

func (c *commitment) recalculate() {


  ...


  quorumMatchIndex := matched[(len(matched)-1)/2]

if quorumMatchIndex > c.commitIndex && quorumMatchIndex >= c.startIndex {

c.commitIndex = quorumMatchIndex

asyncNotifyCh(c.commitCh)


}


}



批量响应的入口

func (r *Raft) processLogs(index uint64, future *logFuture) {

...

r.processLog(&future.log, future)

...


}



响应客户端的key put

...

if future != nil {

future.respond(nil)

}

...

推荐阅读更多精彩内容