func TestBuildInfo(t *testing.T) {
	l := localcluster.Create(1, stopper)
	l.Start()
	defer l.AssertAndStop(t)

	util.SucceedsWithin(t, 10*time.Second, func() error {
		select {
		case <-stopper:
			t.Fatalf("interrupted")
			return nil
		case <-time.After(200 * time.Millisecond):
		}
		var r struct {
			BuildInfo map[string]string
		}
		if err := l.Nodes[0].GetJSON("", "/_status/details/local", &r); err != nil {
			return err
		}
		for _, key := range []string{"goVersion", "tag", "time", "dependencies"} {
			if val, ok := r.BuildInfo[key]; !ok {
				t.Errorf("build info missing for \"%s\"", key)
			} else if val == "" {
				t.Errorf("build info not set for \"%s\"", key)
			}
		}
		return nil
	})
}

func TestRangeReplication(t *testing.T) {
	l := localcluster.Create(*numNodes, stopper)
	l.Start()
	defer l.Stop()

	checkRangeReplication(t, l, 20*time.Second)
}

// TestGossipRestart verifies that the gossip network can be
// re-bootstrapped after a time when all nodes were down
// simultaneously.
func TestGossipRestart(t *testing.T) {
	l := localcluster.Create(*numNodes, stopper)
	l.Start()
	defer l.Stop()

	log.Infof("waiting for initial gossip connections")
	checkGossip(t, l, 20*time.Second, hasPeers(len(l.Nodes)))
	checkGossip(t, l, time.Second, hasClusterID)
	checkGossip(t, l, time.Second, hasSentinel)

	// The replication of the first range is important: as long as the
	// first range only exists on one node, that node can trivially
	// acquire the leader lease. Once the range is replicated, however,
	// nodes must be able to discover each other over gossip before the
	// lease can be acquired.
	log.Infof("waiting for range replication")
	checkRangeReplication(t, l, 10*time.Second)

	log.Infof("stopping all nodes")
	for _, node := range l.Nodes {
		node.Stop(5)
	}

	log.Infof("restarting all nodes")
	for _, node := range l.Nodes {
		node.Restart(5)
	}

	log.Infof("waiting for gossip to be connected")
	checkGossip(t, l, 20*time.Second, hasPeers(len(l.Nodes)))
	checkGossip(t, l, time.Second, hasClusterID)
	checkGossip(t, l, time.Second, hasSentinel)
}

// TestPut starts up an N node cluster and runs N workers that write
// to independent keys.
func TestPut(t *testing.T) {
	l := localcluster.Create(*numNodes, stopper)
	l.Start()
	defer l.Stop()

	db, dbStopper := makeDBClient(t, l, 0)
	defer dbStopper.Stop()
	if err := configutil.SetDefaultRangeMaxBytes(db, *rangeMaxBytes); err != nil {
		t.Fatal(err)
	}
	checkRangeReplication(t, l, 20*time.Second)

	errs := make(chan error, *numNodes)
	start := time.Now()
	deadline := start.Add(*duration)
	var count int64
	for i := 0; i < *numNodes; i++ {
		go func() {
			r, _ := randutil.NewPseudoRand()
			value := randutil.RandBytes(r, 8192)

			for time.Now().Before(deadline) {
				k := atomic.AddInt64(&count, 1)
				v := value[:r.Intn(len(value))]
				if err := db.Put(fmt.Sprintf("%08d", k), v); err != nil {
					errs <- err
					return
				}
			}
			errs <- nil
		}()
	}

	for i := 0; i < *numNodes; {
		select {
		case <-stopper:
			t.Fatalf("interrupted")
		case err := <-errs:
			if err != nil {
				t.Fatal(err)
			}
			i++
		case <-time.After(1 * time.Second):
			// Periodically print out progress so that we know the test is still
			// running.
			log.Infof("%d", atomic.LoadInt64(&count))
		}
	}

	elapsed := time.Since(start)
	log.Infof("%d %.1f/sec", count, float64(count)/elapsed.Seconds())
}

// TestStatusServer starts up an N node cluster and tests the status server on
// each node.
func TestStatusServer(t *testing.T) {
	t.Skipf("TODO(Bram): Test is flaky - fix it.")
	l := localcluster.Create(*numNodes, stopper)
	l.ForceLogging = true
	l.Start()
	defer l.Stop()
	checkRangeReplication(t, l, 20*time.Second)

	client := &http.Client{
		Timeout: 200 * time.Millisecond,
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}

	// Get the ids for each node.
	idMap := make(map[string]string)
	for _, node := range l.Nodes {
		body := get(t, client, node, "/_status/details/local")
		var detail details
		if err := json.Unmarshal(body, &detail); err != nil {
			t.Fatalf("unable to parse details - %s", err)
		}
		idMap[node.ID] = detail.NodeID.String()
	}

	// Check local response for the every node.
	for _, node := range l.Nodes {
		checkNode(t, client, node, idMap[node.ID], "local", idMap[node.ID])
		get(t, client, node, "/_status/nodes")
		get(t, client, node, "/_status/stores")
	}

	// Proxy from the first node to the last node.
	firstNode := l.Nodes[0]
	lastNode := l.Nodes[len(l.Nodes)-1]
	firstID := idMap[firstNode.ID]
	lastID := idMap[lastNode.ID]
	checkNode(t, client, firstNode, firstID, lastID, lastID)

	// And from the last node to the first node.
	checkNode(t, client, lastNode, lastID, firstID, firstID)

	// And from the last node to the last node.
	checkNode(t, client, lastNode, lastID, lastID, lastID)
}

// TestGossipRestart verifies that the gossip network can be
// re-bootstrapped after a time when all nodes were down
// simultaneously.
func TestGossipRestart(t *testing.T) {
	l := localcluster.Create(*numNodes, stopper)
	l.Start()
	defer l.AssertAndStop(t)

	log.Infof("waiting for initial gossip connections")
	checkGossip(t, l, 20*time.Second, hasPeers(len(l.Nodes)))
	checkGossip(t, l, time.Second, hasClusterID)
	checkGossip(t, l, time.Second, hasSentinel)

	// The replication of the first range is important: as long as the
	// first range only exists on one node, that node can trivially
	// acquire the leader lease. Once the range is replicated, however,
	// nodes must be able to discover each other over gossip before the
	// lease can be acquired.
	log.Infof("waiting for range replication")
	checkRangeReplication(t, l, 10*time.Second)

	log.Infof("killing all nodes")
	for _, node := range l.Nodes {
		node.Kill()
	}

	log.Infof("restarting all nodes")
	for _, node := range l.Nodes {
		node.Restart(5)
	}

	log.Infof("waiting for gossip to be connected")
	checkGossip(t, l, 20*time.Second, hasPeers(len(l.Nodes)))
	checkGossip(t, l, time.Second, hasClusterID)
	checkGossip(t, l, time.Second, hasSentinel)

	for i := range l.Nodes {
		db, dbStopper := makeDBClient(t, l, i)
		if kv, err := db.Inc("count", 1); err != nil {
			t.Fatal(err)
		} else if v := kv.ValueInt(); v != int64(i+1) {
			t.Fatalf("unexpected value %d for write #%d (expected %d)", v, i, i+1)
		}
		dbStopper.Stop()
	}

}

// TestStatusServer starts up an N node cluster and tests the status server on
// each node.
func TestStatusServer(t *testing.T) {
	l := localcluster.Create(*numNodes, stopper)
	l.ForceLogging = true
	l.Start()
	defer l.Stop()
	checkRangeReplication(t, l, 20*time.Second)

	// Get the ids for each node.
	idMap := make(map[string]string)
	for _, node := range l.Nodes {
		body := get(t, node, "/_status/details/local")
		var detail details
		if err := json.Unmarshal(body, &detail); err != nil {
			t.Fatalf("unable to parse details - %s", err)
		}
		idMap[node.ID] = detail.NodeID.String()
	}

	// Check local response for the every node.
	for _, node := range l.Nodes {
		checkNode(t, node, idMap[node.ID], "local", idMap[node.ID])
		get(t, node, "/_status/nodes")
		get(t, node, "/_status/stores")
	}

	// Proxy from the first node to the last node.
	firstNode := l.Nodes[0]
	lastNode := l.Nodes[len(l.Nodes)-1]
	firstID := idMap[firstNode.ID]
	lastID := idMap[lastNode.ID]
	checkNode(t, firstNode, firstID, lastID, lastID)

	// And from the last node to the first node.
	checkNode(t, lastNode, lastID, firstID, firstID)

	// And from the last node to the last node.
	checkNode(t, lastNode, lastID, lastID, lastID)
}

func TestGossipPeerings(t *testing.T) {
	l := localcluster.Create(*numNodes, stopper)
	l.Start()
	defer l.AssertAndStop(t)

	checkGossip(t, l, 20*time.Second, hasPeers(len(l.Nodes)))

	// Restart the first node.
	log.Infof("restarting node 0")
	if err := l.Nodes[0].Restart(5); err != nil {
		t.Fatal(err)
	}
	checkGossip(t, l, 20*time.Second, hasPeers(len(l.Nodes)))

	// Restart another node.
	rand.Seed(randutil.NewPseudoSeed())
	pickedNode := rand.Intn(len(l.Nodes)-1) + 1
	log.Infof("restarting node %d", pickedNode)
	if err := l.Nodes[pickedNode].Restart(5); err != nil {
		t.Fatal(err)
	}
	checkGossip(t, l, 20*time.Second, hasPeers(len(l.Nodes)))
}

// TestChaos starts up a cluster and, for each node, a worker writing to
// independent keys, while nodes are being killed and restarted continuously.
// The test measures not write performance, but cluster recovery.
func TestChaos(t *testing.T) {
	t.Skip("TODO(tschottdorf): currently unstable")
	l := localcluster.Create(*numNodes, stopper)
	l.Start()
	defer l.AssertAndStop(t)

	checkRangeReplication(t, l, 20*time.Second)

	errs := make(chan error, *numNodes)
	start := time.Now()
	deadline := start.Add(*duration)
	var count int64
	counts := make([]int64, *numNodes)
	clients := make([]struct {
		sync.RWMutex
		db      *client.DB
		stopper *stop.Stopper
	}, *numNodes)

	initClient := func(i int) {
		db, dbStopper := makeDBClient(t, l, i)
		if clients[i].stopper != nil {
			clients[i].stopper.Stop()
		}
		clients[i].db, clients[i].stopper = db, dbStopper
	}

	for i := 0; i < *numNodes; i++ {
		initClient(i)
		go func(i int) {
			r, _ := randutil.NewPseudoRand()
			value := randutil.RandBytes(r, 8192)

			for time.Now().Before(deadline) {
				clients[i].RLock()
				k := atomic.AddInt64(&count, 1)
				atomic.AddInt64(&counts[i], 1)
				v := value[:r.Intn(len(value))]
				if err := clients[i].db.Put(fmt.Sprintf("%08d", k), v); err != nil {
					// These originate from DistSender when, for example, the
					// leader is down. With more realistic retry options, we
					// should probably not see them.
					if _, ok := err.(*roachpb.SendError); ok {
						log.Warning(err)
					} else {
						errs <- err
						clients[i].RUnlock()
						return
					}
				}
				clients[i].RUnlock()
			}
			errs <- nil
		}(i)
	}

	teardown := make(chan struct{})
	defer func() {
		<-teardown
		for i := range clients {
			clients[i].stopper.Stop()
			clients[i].stopper = nil
		}
	}()

	// Chaos monkey.
	go func() {
		defer close(teardown)
		rnd, seed := randutil.NewPseudoRand()
		log.Warningf("monkey starts (seed %d)", seed)
		for round := 1; time.Now().Before(deadline); round++ {
			select {
			case <-stopper:
				return
			default:
			}
			nodes := rnd.Perm(*numNodes)[:rnd.Intn(*numNodes)+1]

			log.Infof("round %d: restarting nodes %v", round, nodes)
			for _, i := range nodes {
				clients[i].Lock()
			}
			for _, i := range nodes {
				log.Infof("restarting %v", i)
				l.Nodes[i].Kill()
				l.Nodes[i].Restart(5)
				initClient(i)
				clients[i].Unlock()
			}
			for cur := atomic.LoadInt64(&count); time.Now().Before(deadline) &&
				atomic.LoadInt64(&count) == cur; time.Sleep(time.Second) {
				l.Assert(t)
				log.Warningf("monkey sleeping while cluster recovers...")
			}
		}
	}()

//.........这里部分代码省略.........

// TestMultiuser starts up an N node cluster and performs various ops
// using different users.
func TestMultiuser(t *testing.T) {
	l := localcluster.Create(*numNodes, stopper)
	l.Start()
	defer l.Stop()

	// Create client certificates for "foo" and "other".
	if err := security.RunCreateClientCert(l.CertsDir, 512, "foo"); err != nil {
		t.Fatal(err)
	}
	if err := security.RunCreateClientCert(l.CertsDir, 512, "other"); err != nil {
		t.Fatal(err)
	}

	checkRangeReplication(t, l, 20*time.Second)

	// Make clients.
	rootClient := makeDBClientForUser(t, l, "root", 0)
	fooClient := makeDBClientForUser(t, l, "foo", 0)
	otherClient := makeDBClientForUser(t, l, "other", 0)

	// Set permissions configs.
	configs := []struct {
		prefix  string
		readers []string
		writers []string
	}{
		// Good to know: "root" is always allowed to read and write.
		{"foo", []string{"foo"}, []string{"foo"}},
		{"foo/public", []string{"foo", "other"}, []string{"foo"}},
		{"tmp", []string{"foo", "other"}, []string{"foo", "other"}},
	}
	for i, cfg := range configs {
		protoConfig := &config.PermConfig{Read: cfg.readers, Write: cfg.writers}
		if err := putPermConfig(rootClient, cfg.prefix, protoConfig); err != nil {
			t.Fatalf("#%d: failed to write config %+v for prefix %q: %v", i, protoConfig, cfg.prefix, err)
		}
	}

	// Write some data. The value is just the key.
	writes := []struct {
		key     string
		db      *client.DB
		success bool
	}{
		{"some-file", rootClient, true}, {"some-file", fooClient, false}, {"some-file", otherClient, false},
		{"foo/a", rootClient, true}, {"foo/a", fooClient, true}, {"foo/a", otherClient, false},
		{"foo/public/b", rootClient, true}, {"foo/public/b", fooClient, true}, {"foo/public/b", otherClient, false},
		{"tmp/c", rootClient, true}, {"tmp/c", fooClient, true}, {"tmp/c", otherClient, true},
	}

	for i, w := range writes {
		err := w.db.Put(w.key, w.key)
		if (err == nil) != w.success {
			t.Errorf("test case #%d: %+v, got err=%v", i, w, err)
		}
	}

	// Read the previously-written files. They all succeeded at least once.
	reads := []struct {
		key     string
		db      *client.DB
		success bool
	}{
		{"some-file", rootClient, true}, {"some-file", fooClient, false}, {"some-file", otherClient, false},
		{"foo/a", rootClient, true}, {"foo/a", fooClient, true}, {"foo/a", otherClient, false},
		{"foo/public/b", rootClient, true}, {"foo/public/b", fooClient, true}, {"foo/public/b", otherClient, true},
		{"tmp/c", rootClient, true}, {"tmp/c", fooClient, true}, {"tmp/c", otherClient, true},
	}

	for i, r := range reads {
		_, err := r.db.Get(r.key)
		if (err == nil) != r.success {
			t.Errorf("test case #%d: %+v, got err=%v", i, r, err)
		}
	}
}

// TestSingleKey stresses the transaction retry machinery by starting
// up an N node cluster and running N workers that are all
// incrementing the value associated with a single key.
func TestSingleKey(t *testing.T) {
	l := localcluster.Create(*numNodes, stopper)
	l.Start()
	defer l.Stop()

	checkRangeReplication(t, l, 20*time.Second)

	// Initialize the value for our test key to zero.
	const key = "test-key"
	db := makeDBClient(t, l, 0)
	if err := db.Put(key, testVal(0)); err != nil {
		t.Fatal(err)
	}

	type result struct {
		err        error
		count      int
		maxLatency time.Duration
	}

	resultCh := make(chan result, *numNodes)
	deadline := time.Now().Add(*duration)
	var expected int64

	// Start up numNodes workers each reading and writing the same
	// key. Each worker is configured to talk to a different node in the
	// cluster.
	for i := 0; i < *numNodes; i++ {
		db := makeDBClient(t, l, i)
		go func() {
			var r result
			for time.Now().Before(deadline) {
				start := time.Now()
				err := db.Txn(func(txn *client.Txn) error {
					r, err := txn.Get(key)
					if err != nil {
						return err
					}
					var v testVal
					if err := v.UnmarshalBinary(r.ValueBytes()); err != nil {
						return err
					}
					b := &client.Batch{}
					b.Put(key, v+1)
					return txn.CommitInBatch(b)
				})
				if err != nil {
					resultCh <- result{err: err}
					return
				}
				atomic.AddInt64(&expected, 1)
				r.count++
				latency := time.Since(start)
				if r.maxLatency < latency {
					r.maxLatency = latency
				}
			}
			resultCh <- r
		}()
	}

	// Verify that none of the workers encountered an error.
	var results []result
	for len(results) < *numNodes {
		select {
		case <-stopper:
			t.Fatalf("interrupted")
		case r := <-resultCh:
			if r.err != nil {
				t.Fatal(r.err)
			}
			results = append(results, r)
		case <-time.After(1 * time.Second):
			// Periodically print out progress so that we know the test is still
			// running.
			log.Infof("%d", atomic.LoadInt64(&expected))
		}
	}

	// Verify the resulting value stored at the key is what we expect.
	r, err := db.Get(key)
	if err != nil {
		t.Fatal(err)
	}
	var v testVal
	if err := v.UnmarshalBinary(r.ValueBytes()); err != nil {
		t.Fatal(err)
	}
	if expected != int64(v) {
		t.Fatalf("expected %d, but found %d", expected, v)
	}
	var maxLatency []time.Duration
	for _, r := range results {
		maxLatency = append(maxLatency, r.maxLatency)
	}
	log.Infof("%d increments: %s", v, maxLatency)
}