tests 10 times ok

src/mr/coordinator.go

@@ -1,95 +1,239 @@
 package mr
 
 import (
+	"errors"
 	"fmt"
 	"log"
 	"net"
 	"net/http"
 	"net/rpc"
 	"os"
+	"sync"
+	"time"
 )
 
+const machineDefaultLastSecond = 3
+
 type Coordinator struct {
 	// Your definitions here.
 	mapTask    []MapTask
-	recudeTask []ReduceTask
+	mapSize    int
+	reduceTask []ReduceTask
+	reduceSize int
 
-	mapTaskQueue    chan int
+	taskQueue chan Task
-	reduceTaskQueue chan int
-	running         chan bool
 
-	lock chan bool
+	lock      sync.Mutex
+	machine   map[int]*Machine
+	machineId int
+
+	running chan bool
+}
+
+func (c *Coordinator) popTask(machine *Machine) {
+	if machine.Task.TaskType == TASK_NONE {
+		return
+	}
+	DPrintf("machine %d pop task %d, task list is %v %v\n", machine.Id,
+		machine.Task.TaskId, machine.Task.MapTask, machine.Task.ReduceTask)
+	c.taskQueue <- machine.Task
+	machine.Task.TaskType = TASK_NONE
+	machine.State = MACHINE_IDLE
+}
+
+func (c *Coordinator) getMachine(id int) *Machine {
+	machine, ok := c.machine[id]
+	if !ok {
+		DPrintf("machine not found at %d\n", id)
+		return nil
+	}
+	return machine
 }
 
 // Your code here -- RPC handlers for the worker to call.
-func (c *Coordinator) GetJob(args *Empty, reply *Task) error {
+func (c *Coordinator) GetTask(args *Machine, reply *Machine) error {
-	select {
+	c.lock.Lock()
-	case id := <-c.mapTaskQueue:
+	defer c.lock.Unlock()
-		reply.MapTask = &c.mapTask[id]
-		reply.MapTask.TaskId = id
-		reply.MapTask.State = PROGRESS
 
-		reply.TaskType = TASK_MAP
+	if c.Done() {
-	case id := <-c.reduceTaskQueue:
+		reply.State = MACHINE_DONE
-		reply.ReduceTask = &c.recudeTask[id]
+		return nil
-		reply.ReduceTask.TaskId = id
-		reply.ReduceTask.State = PROGRESS
-
-		entries, err := os.ReadDir(".")
-		if err != nil {
-			log.Fatal("read dir error")
-		}
-		var mapId, reduceId int
-		for _, entry := range entries {
-
-			n, _ := fmt.Sscanf(entry.Name(), "mr-%d-%d", &mapId, &reduceId)
-			if n == 2 && reduceId == reply.ReduceTask.TaskId {
-				// log.Printf("reduce task %d find file %s", reply.ReduceTask.TaskId, entry.Name())
-				reply.ReduceTask.Fnames = append(reply.ReduceTask.Fnames, entry.Name())
-			}
-		}
-
-		reply.TaskType = TASK_REDUCE
 	}
+
+	machine := c.getMachine(args.Id)
+	if machine == nil {
+		reply.State = MACHINE_INIT
+		return nil
+	}
+
+	if machine.Task.TaskType != TASK_NONE {
+		return errors.New("task is not none")
+	}
+
+	c.lock.Unlock()
+	task, ok := <-c.taskQueue
+	c.lock.Lock()
+	if !ok {
+		reply.State = MACHINE_DONE
+		return nil
+	}
+	machine.Task = task
+	machine.State = MACHINE_READY
+
+	DPrintf("machine %d get task %d, task list is %v %v\n", machine.Id,
+		machine.Task.TaskId, machine.Task.MapTask, machine.Task.ReduceTask)
+	*reply = *machine
 	return nil
 }
 
-func (c *Coordinator) SubmitJob(args *Task, reply *Empty) error {
+func (c *Coordinator) genReduceTask() {
-	<-c.lock
+	var mapId, reduceId int
-	defer func() { c.lock <- true }()
+	entries, err := os.ReadDir(".")
+	if err != nil {
+		log.Fatal("read dir error")
+	}
+	for _, entry := range entries {
+		n, _ := fmt.Sscanf(entry.Name(), "mr-%d-%d", &mapId, &reduceId)
+		if n == 2 && reduceId < c.reduceSize && mapId >= 0 {
+			// log.Printf("reduce task %d find file %s", reply.ReduceTask.TaskId, entry.Name())
+			c.reduceTask[reduceId].Fnames = append(c.reduceTask[reduceId].Fnames, entry.Name())
+		}
+	}
+}
 
-	switch args.TaskType {
+func (c *Coordinator) SubmitTask(args *Machine, reply *Machine) error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if c.Done() {
+		reply.State = MACHINE_DONE
+		return nil
+	}
+
+	machine := c.getMachine(args.Id)
+	if machine == nil {
+		reply.State = MACHINE_INIT
+		return nil
+	}
+	task := machine.Task
+
+	switch task.TaskType {
 	case TASK_MAP:
-		// log.Print("submit map task")
+		// log.Printf("submit map task %d\n", task.TaskId)
-		c.mapTask[args.MapTask.TaskId].State = COMPELETED
+		c.mapTask[task.TaskId].done = true
 		for _, task := range c.mapTask {
-			if task.State != COMPELETED {
+			if !task.done {
-				return nil
+				goto END
 			}
 		}
 
-		log.Print("start reduce task")
+		c.genReduceTask()
-		for _, task := range c.recudeTask {
+		DPrintf("end map task\n")
-			c.reduceTaskQueue <- task.TaskId
+		go func() {
-		}
+			for i, task := range c.reduceTask {
+				c.taskQueue <- Task{
+					MachineId:  -1,
+					TaskId:     i,
+					TaskType:   TASK_REDUCE,
+					ReduceTask: &task,
+					MapTask:    nil,
+				}
+			}
+		}()
+		// log.Printf("task queue is %+v %+v\n", c.reduceTask, c.taskQueue)
 	case TASK_REDUCE:
 		// log.Print("submit reduce task")
-		c.recudeTask[args.ReduceTask.TaskId].State = COMPELETED
+		c.reduceTask[task.TaskId].done = true
-		for _, task := range c.recudeTask {
+		for _, task := range c.reduceTask {
-			if task.State != COMPELETED {
+			if !task.done {
-				return nil
+				goto END
 			}
 		}
 
-		log.Print("end")
+		DPrintf("end reduce task\n")
-		// close(c.reduceTaskQueue)
+		for _, m := range c.machine {
-		// close(c.mapTaskQueue)
+			delete(c.machine, m.Id)
+		}
+		close(c.taskQueue)
+
 		<-c.running
 		c.running <- false
+	case TASK_NONE:
+		log.Print("submit tasknone")
 	}
+
+END:
+	machine.State = MACHINE_IDLE
+	machine.Task.TaskType = TASK_NONE
+	*reply = *machine
 	return nil
 }
 
+func (c *Coordinator) FlushTimeout(args int, reply *Empty) error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if c.Done() {
+		return nil
+	}
+
+	machine := c.getMachine(args)
+	if machine == nil {
+		return errors.New("flush machine not found")
+	}
+
+	machine.LastSecond = machineDefaultLastSecond
+	return nil
+}
+
+func (c *Coordinator) UpdateState(args *Machine, reply *Machine) error {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	if args.State == MACHINE_INIT {
+		newId := c.machineId
+		c.machineId += 1
+
+		machine := MakeMachine(newId, machineDefaultLastSecond)
+		machine.State = MACHINE_IDLE
+
+		*reply = *machine
+		c.machine[newId] = machine
+		// log.Printf("machine %d init map is %v\n", newId, c.machine)
+		return nil
+	}
+
+	machine, ok := c.machine[args.Id]
+	if !ok {
+		*reply = *args
+		reply.State = MACHINE_INIT
+		return nil
+	}
+
+	machine.LastSecond = machineDefaultLastSecond
+	*reply = *machine
+	return nil
+}
+
+func (c *Coordinator) deamon() {
+	for {
+		time.Sleep(time.Second)
+
+		c.lock.Lock()
+		for _, m := range c.machine {
+			if m.LastSecond > 0 {
+				m.LastSecond -= 1
+			}
+			if m.LastSecond == 0 {
+				new_m := Machine{}
+				new_m = *m
+				go c.popTask(&new_m)
+				DPrintf("delete machine %d\n", m.Id)
+				delete(c.machine, m.Id)
+			}
+		}
+		c.lock.Unlock()
+	}
+}
+
 // an example RPC handler.
 //
 // the RPC argument and reply types are defined in rpc.go.
@@ -130,40 +274,51 @@ func (c *Coordinator) Done() bool {
 // nReduce is the number of reduce tasks to use.
 func MakeCoordinator(files []string, nReduce int) *Coordinator {
 	c := Coordinator{
-		mapTask:         make([]MapTask, len(files)),
+		mapTask: make([]MapTask, len(files)),
-		recudeTask:      make([]ReduceTask, nReduce),
+		mapSize: len(files),
-		mapTaskQueue:    make(chan int, len(files)),
+
-		reduceTaskQueue: make(chan int, nReduce),
+		reduceTask: make([]ReduceTask, nReduce),
-		running:         make(chan bool, 1),
+		reduceSize: nReduce,
-		lock:            make(chan bool, 1),
+
+		taskQueue: make(chan Task),
+
+		machine:   make(map[int]*Machine),
+		machineId: 1,
+		lock:      sync.Mutex{},
+		running:   make(chan bool, 1),
 	}
 
 	// Your code here.
 	c.running <- true
-	c.lock <- true
 
 	for i, file := range files {
 		c.mapTask[i] = MapTask{
-			Fname:     file,
+			Fname:   file,
-			State:     IDLE,
+			NReduce: nReduce,
-			NReduce:   nReduce,
+			done:    false,
-			TaskId:    i,
-			MachineId: -1,
 		}
 	}
 
 	for i := range nReduce {
-		c.recudeTask[i] = ReduceTask{
+		c.reduceTask[i] = ReduceTask{
-			Fnames:    make([]string, 0),
+			Fnames: make([]string, 0),
-			State:     IDLE,
+			done:   false,
-			TaskId:    i,
-			MachineId: -1,
 		}
 	}
 
-	for _, task := range c.mapTask {
+	go func() {
-		c.mapTaskQueue <- task.TaskId
+		for i, task := range c.mapTask {
-	}
+			c.taskQueue <- Task{
+				MachineId:  -1,
+				TaskType:   TASK_MAP,
+				MapTask:    &task,
+				ReduceTask: nil,
+				TaskId:     i,
+			}
+		}
+	}()
+
+	go c.deamon()
 
 	c.server()
 	return &c

src/mr/rpc.go

@@ -7,6 +7,7 @@ package mr
 //
 
 import (
+	"fmt"
 	"os"
 	"strconv"
 )
@@ -26,42 +27,71 @@ type ExampleReply struct {
 
 // Add your RPC definitions here.
 
-type State int
+func DPrintf(format string, a ...any) (int, error) {
+	const Debug = false
+	var n int
+	var err error
+	if Debug {
+		n, err = fmt.Printf(format, a...)
+	}
+	return n, err
+}
+
+type MachineState int
 
 const (
-	IDLE State = iota
+	MACHINE_INIT MachineState = iota
-	PROGRESS
+	MACHINE_IDLE
-	COMPELETED
+	MACHINE_READY
+	MACHINE_RUNNING
+	MACHINE_DONE
 )
 
+type Machine struct {
+	State      MachineState
+	Id         int
+	LastSecond int
+	Task       Task
+}
+
+func MakeMachine(id int, lastSecond int) *Machine {
+	return &Machine{
+		State:      MACHINE_INIT,
+		Id:         id,
+		LastSecond: lastSecond,
+		Task: Task{
+			TaskType:   TASK_NONE,
+			MapTask:    nil,
+			ReduceTask: nil,
+		},
+	}
+}
+
 type TaskType int
 
 const (
-	TASK_IDLE TaskType = iota
+	TASK_NONE TaskType = iota
 	TASK_MAP
 	TASK_REDUCE
-	TASK_DONE
 )
 
 type Task struct {
+	TaskId     int
+	MachineId  int
 	TaskType   TaskType
 	MapTask    *MapTask
 	ReduceTask *ReduceTask
 }
 
 type MapTask struct {
-	Fname     string
+	Fname   string
-	State     State
+	NReduce int
-	NReduce   int
+	done    bool
-	TaskId    int
-	MachineId int
 }
 
 type ReduceTask struct {
-	Fnames    []string
+	Fnames []string
-	State     State
+	done   bool
-	MachineId int
-	TaskId    int
 }
 
 type Empty struct {

src/mr/worker.go

@@ -9,6 +9,8 @@ import (
 	"net/rpc"
 	"os"
 	"sort"
+	"sync/atomic"
+	"time"
 )
 
 // Map functions return a slice of KeyValue.
@@ -34,32 +36,84 @@ func Worker(mapf func(string, string) []KeyValue,
 	// uncomment to send the Example RPC to the coordinator.
 	// CallExample()
 
-	empty := Empty{}
+	machine := MakeMachine(-1, -1)
-	for {
+	var id atomic.Int32
-		reply := Task{}
-		ok := call("Coordinator.GetJob", &empty, &reply)
-		if !ok {
-			break
-		}
 
-		switch reply.TaskType {
+	for {
-		case TASK_MAP:
+		reply := Machine{}
-			kva := workMap(mapf, reply.MapTask)
+		state := machine.State
-			saveMap(kva, reply.MapTask)
+		switch state {
-			call("Coordinator.SubmitJob", &reply, &empty)
+		case MACHINE_INIT:
-		case TASK_REDUCE:
+			// log.Printf("worker init %d\n", machine.Id)
-			workRecude(reducef, reply.ReduceTask)
+			if ok := call("Coordinator.UpdateState", machine, &reply); !ok {
-			call("Coordinator.SubmitJob", &reply, &empty)
+				reply.State = MACHINE_DONE
-		case TASK_IDLE:
+			} else {
-		case TASK_DONE:
+				id.Store(int32(reply.Id))
-			goto END
+				go func() {
+					for {
+						time.Sleep(time.Second)
+						rid := id.Load()
+						if ok := call("Coordinator.FlushTimeout", rid, &Empty{}); !ok {
+							log.Printf("FlushTimeout error %v\n", ok)
+						}
+					}
+				}()
+			}
+			machine = &reply
+		case MACHINE_READY:
+			if ok := runOnceTask(machine, mapf, reducef); !ok {
+				machine.State = MACHINE_IDLE
+			}
+		case MACHINE_IDLE:
+			// log.Printf("worker idle %d\n", machine.Id)
+			machine.Task.TaskType = TASK_NONE
+			if ok := call("Coordinator.GetTask", machine, &reply); !ok {
+				reply.State = MACHINE_DONE
+			}
+			machine = &reply
+		case MACHINE_DONE:
+			return
+		default:
+			goto SLEEP
 		}
+		continue
+	SLEEP:
+		time.Sleep(time.Second)
 	}
-END:
 }
 
-func workMap(mapf func(string, string) []KeyValue, params *MapTask) []KeyValue {
+func runOnceTask(machine *Machine,
-	filename := params.Fname
+	mapf func(string, string) []KeyValue,
+	reducef func(string, []string) string) bool {
+	machine.State = MACHINE_RUNNING
+	task := &machine.Task
+
+	switch task.TaskType {
+	case TASK_MAP:
+		kva := workMap(mapf, task)
+		saved_fname := saveMap(kva, task)
+		if saved_fname == nil {
+			return false
+		}
+	case TASK_REDUCE:
+		workRecude(reducef, task)
+	default:
+		log.Printf("unknown task type %d", task.TaskType)
+		return false
+	}
+
+	// log.Printf("submit task %d %+v %+v", task.TaskId, task.MapTask, task.ReduceTask)
+	reply := Machine{}
+	if ok := call("Coordinator.SubmitTask", machine, &reply); !ok {
+		return false
+	}
+	*machine = reply
+
+	return true
+}
+
+func workMap(mapf func(string, string) []KeyValue, params *Task) []KeyValue {
+	filename := params.MapTask.Fname
 
 	file, err := os.Open(filename)
 	if err != nil {
@@ -74,10 +128,11 @@ func workMap(mapf func(string, string) []KeyValue, params *MapTask) []KeyValue {
 	return kva
 }
 
-func saveMap(kva []KeyValue, params *MapTask) []string {
+func saveMap(kva []KeyValue, params *Task) []string {
-	fnames := make([]string, params.NReduce)
+	nReduce := params.MapTask.NReduce
-	files := make([]*os.File, params.NReduce)
+	fnames := make([]string, nReduce)
-	encodes := make([]*json.Encoder, params.NReduce)
+	files := make([]*os.File, nReduce)
+	encodes := make([]*json.Encoder, nReduce)
 
 	for i := range fnames {
 		fnames[i] = fmt.Sprintf("mr-%d-%d", params.TaskId, i)
@@ -99,7 +154,7 @@ func saveMap(kva []KeyValue, params *MapTask) []string {
 	}()
 
 	for _, kv := range kva {
-		enc := encodes[ihash(kv.Key)%params.NReduce]
+		enc := encodes[ihash(kv.Key)%nReduce]
 		if err := enc.Encode(&kv); err != nil {
 			log.Fatalf("cannot encode %v", kv)
 			break
@@ -117,11 +172,12 @@ func (a ByKey) Len() int           { return len(a) }
 func (a ByKey) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 func (a ByKey) Less(i, j int) bool { return a[i].Key < a[j].Key }
 
-func workRecude(reducef func(string, []string) string, params *ReduceTask) {
+func workRecude(reducef func(string, []string) string, params *Task) {
 	reduceId := params.TaskId
+	fnames := params.ReduceTask.Fnames
 	intermediate := []KeyValue{}
 
-	for _, fname := range params.Fnames {
+	for _, fname := range fnames {
 		file, err := os.Open(fname)
 		if err != nil {
 			log.Fatal("can't open file")