init

node/manager.go

@@ -0,0 +1,95 @@
+package node
+
+import (
+	"errors"
+	"sort"
+	"time"
+)
+
+type NodeManager struct {
+	root      *PathNode
+	current   *PathNode
+	scan_time time.Duration
+}
+
+func NewNodeManager() *NodeManager {
+	root := NewPathNode("")
+	return &NodeManager{
+		root:      root,
+		current:   root,
+		scan_time: time.Duration(0),
+	}
+}
+
+func (npm *NodeManager) VaildRoot() bool {
+	return npm.root.VaildPath()
+}
+
+func (npm *NodeManager) SetRootPath(path string) bool {
+	npm.root.path = path
+	return npm.root.VaildPath()
+}
+
+func (npm *NodeManager) GetScanTime() time.Duration {
+	return npm.scan_time
+}
+
+func (npm *NodeManager) SaveToFile() error {
+	return npm.root.ToJSON("saved.json.gz", JSON_GZ)
+}
+
+func (npm *NodeManager) LoadFromFile() error {
+	return npm.root.FromJSON("saved.json.gz", JSON_GZ)
+}
+
+func (npm *NodeManager) Scan() error {
+	if !npm.VaildRoot() {
+		return errors.New("root path not vaild")
+	}
+	npm.root.ClearWithoutFlush()
+	start := time.Now()
+	err := npm.root.FastChanIterScanNode(0)
+	if err != nil {
+		return err
+	}
+	npm.current = npm.root
+	npm.scan_time = time.Since(start)
+	return nil
+}
+
+func (npm *NodeManager) GetChildrenWithSorted() []*PathNode {
+	nodes := npm.current.GetChildren()
+	sort.Slice(nodes, func(i, j int) bool {
+		if nodes[i].size > nodes[j].size {
+			return true
+		} else if nodes[i].size < nodes[j].size {
+			return false
+		}
+		return nodes[i].path > nodes[j].path
+	})
+	return nodes
+}
+
+func (npm *NodeManager) GetCurrentNode() *PathNode {
+	return npm.current
+}
+
+func (npm *NodeManager) GoIn(pathName string) error {
+	res := npm.current.GetChild(pathName)
+	if res == nil {
+		return errors.New("children not found")
+	}
+	if res.node_type != DIR {
+		return errors.New("not a directory")
+	}
+	npm.current = res
+	return nil
+}
+
+func (npm *NodeManager) GoOut() error {
+	if npm.current.parent == nil {
+		return errors.New("you can't go out of root")
+	}
+	npm.current = npm.current.parent
+	return nil
+}

node/node.go

@@ -0,0 +1,104 @@
+package node
+
+import (
+	"os"
+)
+
+// PathNodeType 表示路径节点类型
+type PathNodeType int
+
+const (
+	DIR PathNodeType = iota
+	FILE
+	UNKNOWN
+
+	// Stat
+	NOSCAN
+)
+
+// PathNode 表示目录树节点
+type PathNode struct {
+	path      string
+	children  map[string]*PathNode
+	parent    *PathNode
+	size      int64
+	count     int
+	node_type PathNodeType
+}
+
+func NewPathNode(path string) *PathNode {
+	return &PathNode{
+		path:      path,
+		children:  make(map[string]*PathNode),
+		parent:    nil,
+		size:      0,
+		count:     0,
+		node_type: NOSCAN,
+	}
+}
+
+func (pn *PathNode) GetPath() string {
+	return pn.path
+}
+
+func (pn *PathNode) GetSize() int64 {
+	return pn.size
+}
+
+func (pn *PathNode) GetSizeByFormat() string {
+	return FormatSize(pn.size)
+}
+
+func (pn *PathNode) GetCount() int {
+	return pn.count
+}
+
+func (pn *PathNode) GetType() PathNodeType {
+	return pn.node_type
+}
+
+func (pn *PathNode) GetChildren() []*PathNode {
+	children := make([]*PathNode, 0, len(pn.children))
+	for _, child := range pn.children {
+		children = append(children, child)
+	}
+	return children
+}
+
+func (pn *PathNode) GetChild(path string) *PathNode {
+	return pn.children[path]
+}
+
+func (pn *PathNode) VaildPath() bool {
+	stat, err := os.Lstat(pn.path)
+	if err != nil {
+		return false
+	}
+	if stat.IsDir() {
+		return true
+	}
+	return false
+}
+
+// linkChild 添加子节点
+func (pn *PathNode) linkChild(child *PathNode) {
+	child.parent = pn
+	pn.children[child.path] = child
+}
+
+func (pn *PathNode) flushNode(diff_size int64, diff_count int) {
+	pn.size += diff_size
+	pn.count += diff_count
+	if pn.parent == nil {
+		return
+	}
+	pn.parent.flushNode(diff_size, diff_count)
+}
+
+func (pn *PathNode) ClearWithoutFlush() {
+	for key, child := range pn.children {
+		child.ClearWithoutFlush()
+		child.parent = nil
+		delete(pn.children, key)
+	}
+}

node/saved.go

@@ -0,0 +1,151 @@
+package node
+
+import (
+	"compress/gzip"
+	"encoding/json"
+	"errors"
+	"io"
+	"os"
+)
+
+type StoreType int
+
+const (
+	JSON_GZ StoreType = iota
+	JSON_NO_IDENT
+	JSON_IDENT
+)
+
+// MarshalJSON 实现自定义序列化
+func (pn *PathNode) MarshalJSON() ([]byte, error) {
+	type Alias PathNode
+	return json.Marshal(&struct {
+		*Alias
+		Path      string               `json:"path"`
+		Children  map[string]*PathNode `json:"children"`
+		Count     int                  `json:"count"`
+		Node_type PathNodeType         `json:"type"`
+		Size      int64                `json:"size"`
+	}{
+		Alias:     (*Alias)(pn),
+		Path:      pn.path,
+		Children:  pn.children,
+		Count:     pn.count,
+		Size:      pn.size,
+		Node_type: pn.node_type,
+	})
+}
+
+// UnmarshalJSON 实现自定义反序列化
+func (pn *PathNode) UnmarshalJSON(data []byte) error {
+	type Alias PathNode
+	temp := &struct {
+		*Alias
+		Path      string               `json:"path"`
+		Children  map[string]*PathNode `json:"children"`
+		Count     int                  `json:"count"`
+		Node_type PathNodeType         `json:"type"`
+		Size      int64                `json:"size"`
+	}{
+		Alias: (*Alias)(pn),
+	}
+
+	if err := json.Unmarshal(data, &temp); err != nil {
+		return err
+	}
+
+	pn.path = temp.Path
+	pn.children = temp.Children
+	pn.count = temp.Count
+	pn.size = temp.Size
+	pn.node_type = temp.Node_type
+	return nil
+}
+
+// ToJSON 将 PathNode 序列化为 JSON 字符串
+func (pn *PathNode) ToJSON(savedPath string, savedType StoreType) error {
+	f, err := os.Create(savedPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	switch savedType {
+	case JSON_GZ:
+		data, err := json.Marshal(pn)
+		if err != nil {
+			return err
+		}
+		gz := gzip.NewWriter(f)
+		defer gz.Close()
+		if _, err := gz.Write(data); err != nil {
+			return err
+		}
+	case JSON_NO_IDENT:
+		data, err := json.Marshal(pn)
+		if err != nil {
+			return err
+		}
+		if _, err = f.Write(data); err != nil {
+			return err
+		}
+	case JSON_IDENT:
+		data, err := json.MarshalIndent(pn, "", "  ")
+		if err != nil {
+			return err
+		}
+		if _, err = f.Write(data); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// FromJSON 从 JSON 字符串反序列化为 PathNode
+func (pn *PathNode) FromJSON(savedPath string, savedType StoreType) error {
+	if pn.parent != nil {
+		return errors.New("PathNode.Parent should be nil")
+	}
+
+	f, err := os.Open(savedPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	var jsonData []byte
+	switch savedType {
+	case JSON_GZ:
+		gzr, err := gzip.NewReader(f)
+		if err != nil {
+			return err
+		}
+		defer gzr.Close()
+		if jsonData, err = io.ReadAll(gzr); err != nil {
+			return err
+		}
+	default:
+		// case JSON_IDENT:
+		// case JSON_NO_IDENT:
+		if jsonData, err = io.ReadAll(f); err != nil {
+			return err
+		}
+	}
+
+	pn.ClearWithoutFlush()
+	if err := json.Unmarshal([]byte(jsonData), pn); err != nil {
+		return err
+	}
+
+	// RestoreParentLinks 恢复 Parent 关系
+	var restoreParentLinks func(pn *PathNode)
+	restoreParentLinks = func(pn *PathNode) {
+		for _, child := range pn.children {
+			child.parent = pn
+			restoreParentLinks(child)
+		}
+	}
+
+	restoreParentLinks(pn)
+	return nil
+}

node/saved_test.go

@@ -0,0 +1,26 @@
+package node
+
+import (
+	"os"
+	"reflect"
+	"testing"
+)
+
+func TestJSON_GZ(t *testing.T) {
+	jsonName := "saved.json.gz"
+	tmpnode := NewPathNode(".")
+	tmpnode.IterScanNode()
+	if err := tmpnode.ToJSON(jsonName, JSON_GZ); err != nil {
+		t.Fatal(err)
+	}
+
+	readNode := NewPathNode("")
+	if err := readNode.FromJSON(jsonName, JSON_GZ); err != nil {
+		t.Fatal(err)
+	}
+	os.Remove(jsonName)
+
+	if !reflect.DeepEqual(tmpnode, readNode) {
+		t.Fatal("not equal")
+	}
+}

node/scan.go

@@ -0,0 +1,193 @@
+package node
+
+import (
+	"os"
+	"runtime"
+	"sync"
+)
+
+func (pn *PathNode) ScanPath() error {
+	stat, err := os.Lstat(pn.path)
+	var nodeType PathNodeType
+
+	if err != nil {
+		// return fmt.Errorf("error os.Lstat: [%w]", err)
+		goto ERR_UNKNOWN
+	}
+
+	if stat.IsDir() {
+		entries, err := os.ReadDir(pn.path)
+		if err != nil {
+			nodeType = UNKNOWN
+		} else {
+			for _, entry := range entries {
+				childNode := NewPathNode(pn.path + "/" + entry.Name())
+				pn.linkChild(childNode)
+			}
+			nodeType = DIR
+		}
+	} else if stat.Mode().IsRegular() {
+		nodeType = FILE
+	} else {
+		nodeType = UNKNOWN
+	}
+
+	pn.node_type = nodeType
+	pn.size = stat.Size()
+	pn.count = 1
+	return nil
+ERR_UNKNOWN:
+	pn.count = 1
+	pn.node_type = UNKNOWN
+	return nil
+}
+
+func (pn *PathNode) IterScanNode() error {
+	err := pn.ScanPath()
+	if err != nil {
+		return err
+	}
+
+	if pn.node_type == DIR {
+		for _, entry := range pn.children {
+			err = entry.IterScanNode()
+			if err != nil {
+				return err
+			}
+		}
+		pn.count--
+		pn.flushNode(0, 1)
+	} else {
+		pn.flushNode(pn.size, pn.count)
+	}
+	return nil
+}
+
+func (pn *PathNode) FastIterScanNode() error {
+	var wg sync.WaitGroup
+	var nodeMu sync.Mutex
+	errChan := make(chan error)
+	// queue := make(chan *PathNode)
+	maxWorker := runtime.NumCPU()
+	curWorker := 1
+	var wkMu sync.Mutex
+
+	var worker func(node *PathNode, isWorker bool)
+	worker = func(node *PathNode, isWorker bool) {
+		err := node.ScanPath()
+		if err != nil {
+			errChan <- err
+			return
+		}
+
+		if node.node_type == DIR {
+			for _, entry := range node.children {
+				wkMu.Lock()
+				if curWorker < maxWorker {
+					curWorker++
+					wkMu.Unlock()
+					wg.Add(1)
+					go worker(entry, true)
+				} else {
+					wkMu.Unlock()
+					worker(entry, false)
+				}
+			}
+		}
+
+		if node.parent != nil {
+			nodeMu.Lock()
+			if node.node_type == DIR {
+				node.parent.flushNode(0, 1)
+			} else {
+				node.parent.flushNode(node.size, node.count)
+			}
+			nodeMu.Unlock()
+		}
+
+		if isWorker {
+			wkMu.Lock()
+			curWorker--
+			wkMu.Unlock()
+			wg.Done()
+		}
+	}
+
+	wg.Add(1)
+	go worker(pn, true)
+
+	go func() {
+		wg.Wait()
+		close(errChan)
+	}()
+
+	for err := range errChan {
+		return err
+	}
+
+	return nil
+}
+
+func (pn *PathNode) FastChanIterScanNode(maxWorker int) error {
+	var wg sync.WaitGroup
+	errChan := make(chan error)
+	if maxWorker <= 0 {
+		maxWorker = runtime.NumCPU()
+	}
+	nodeCh := make(chan bool, 1)
+	defer close(nodeCh)
+
+	wkCh := make(chan bool, maxWorker)
+	defer close(wkCh)
+
+	var worker func(node *PathNode, isWorker bool)
+	worker = func(node *PathNode, isWorker bool) {
+		err := node.ScanPath()
+		if err != nil {
+			errChan <- err
+			return
+		}
+
+		if node.node_type == DIR {
+			for _, entry := range node.children {
+				select {
+				case wkCh <- true:
+					wg.Add(1)
+					go worker(entry, true)
+				default:
+					worker(entry, false)
+				}
+			}
+		}
+
+		if node.parent != nil {
+			nodeCh <- true
+			if node.node_type == DIR {
+				node.parent.flushNode(0, 1)
+			} else {
+				node.parent.flushNode(node.size, node.count)
+			}
+			<-nodeCh
+		}
+
+		if isWorker {
+			<-wkCh
+			wg.Done()
+		}
+	}
+
+	wg.Add(1)
+	wkCh <- true
+	go worker(pn, true)
+
+	go func() {
+		wg.Wait()
+		close(errChan)
+	}()
+
+	for err := range errChan {
+		return err
+	}
+
+	return nil
+}

node/scan_test.go

@@ -0,0 +1,78 @@
+package node
+
+import (
+	"os"
+	"testing"
+)
+
+// 辅助函数：创建临时文件系统
+func setupTestFileSystem(t *testing.T) (string, *PathNode) {
+	tempDir, err := os.MkdirTemp("", "test-scan")
+	if err != nil {
+		t.Fatalf("Failed to create temp dir: %v", err)
+	}
+
+	// 创建子目录和文件
+	subDir := tempDir + "/subdir"
+	err = os.Mkdir(subDir, 0755)
+	if err != nil {
+		t.Fatalf("Failed to create sub directory: %v", err)
+	}
+
+	tempFile, err := os.CreateTemp(subDir, "testfile")
+	if err != nil {
+		t.Fatalf("Failed to create temp file: %v", err)
+	}
+	tempFile.Close()
+
+	// 创建 PathNode
+	pn := NewPathNode(tempDir)
+
+	return tempDir, pn
+}
+
+// 辅助函数：验证扫描结果
+func verifyScanResult(t *testing.T, pn *PathNode) {
+	if pn.node_type != DIR {
+		t.Errorf("Expected node type DIR, got %v", pn.node_type)
+	}
+	if pn.count != 3 { // 1 for the subdir, 1 for the file
+		t.Errorf("Expected count 3, got %v", pn.count)
+	}
+}
+
+func TestIterScanNode(t *testing.T) {
+	tempDir, pn := setupTestFileSystem(t)
+	defer os.RemoveAll(tempDir)
+
+	err := pn.IterScanNode()
+	if err != nil {
+		t.Errorf("IterScanNode failed: %v", err)
+	}
+
+	verifyScanResult(t, pn)
+}
+
+func TestFastIterScanNode(t *testing.T) {
+	tempDir, pn := setupTestFileSystem(t)
+	defer os.RemoveAll(tempDir)
+
+	err := pn.FastIterScanNode()
+	if err != nil {
+		t.Errorf("FastIterScanNode failed: %v", err)
+	}
+
+	verifyScanResult(t, pn)
+}
+
+func TestFastChanIterScanNode(t *testing.T) {
+	tempDir, pn := setupTestFileSystem(t)
+	defer os.RemoveAll(tempDir)
+
+	err := pn.FastChanIterScanNode(0) // 使用默认的 maxWorker
+	if err != nil {
+		t.Errorf("FastChanIterScanNode failed: %v", err)
+	}
+
+	verifyScanResult(t, pn)
+}

node/utils.go

@@ -0,0 +1,26 @@
+package node
+
+import "fmt"
+
+// BytesToHumanReadable 将字节数转换为人类可读的字符串
+func BytesToHumanReadable(num float64, suffix string) string {
+	for _, unit := range []string{"", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"} {
+		if num < 1024 {
+			return fmt.Sprintf("%3.1f %s%s", float64(num), unit, suffix)
+		}
+		num /= 1024
+	}
+	return fmt.Sprintf("%.1f Yi%s", float64(num), suffix)
+}
+
+// FormatSize 格式化大小为人类可读的格式
+func FormatSize(size int64) string {
+	res := BytesToHumanReadable(float64(size), "B") //.PadLeft(10)
+	return fmt.Sprintf("%10s", res)
+}
+
+func (pn *PathNode) ShowChildrenSimple() {
+	for _, child := range pn.children {
+		fmt.Printf("%d %s %s %d\n", child.node_type, FormatSize(child.size), child.path, child.count)
+	}
+}