mkmerge.go 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521
  1. // Copyright 2020 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // +build ignore
  5. // mkmerge.go parses generated source files and merges common
  6. // consts, funcs, and types into a common source file, per GOOS.
  7. //
  8. // Usage:
  9. // $ go run mkmerge.go -out MERGED FILE [FILE ...]
  10. //
  11. // Example:
  12. // # Remove all common consts, funcs, and types from zerrors_linux_*.go
  13. // # and write the common code into zerrors_linux.go
  14. // $ go run mkmerge.go -out zerrors_linux.go zerrors_linux_*.go
  15. //
  16. // mkmerge.go performs the merge in the following steps:
  17. // 1. Construct the set of common code that is idential in all
  18. // architecture-specific files.
  19. // 2. Write this common code to the merged file.
  20. // 3. Remove the common code from all architecture-specific files.
  21. package main
  22. import (
  23. "bufio"
  24. "bytes"
  25. "flag"
  26. "fmt"
  27. "go/ast"
  28. "go/format"
  29. "go/parser"
  30. "go/token"
  31. "io"
  32. "io/ioutil"
  33. "log"
  34. "os"
  35. "path"
  36. "path/filepath"
  37. "regexp"
  38. "strconv"
  39. "strings"
  40. )
  41. const validGOOS = "aix|darwin|dragonfly|freebsd|linux|netbsd|openbsd|solaris"
  42. // getValidGOOS returns GOOS, true if filename ends with a valid "_GOOS.go"
  43. func getValidGOOS(filename string) (string, bool) {
  44. matches := regexp.MustCompile(`_(` + validGOOS + `)\.go$`).FindStringSubmatch(filename)
  45. if len(matches) != 2 {
  46. return "", false
  47. }
  48. return matches[1], true
  49. }
  50. // codeElem represents an ast.Decl in a comparable way.
  51. type codeElem struct {
  52. tok token.Token // e.g. token.CONST, token.TYPE, or token.FUNC
  53. src string // the declaration formatted as source code
  54. }
  55. // newCodeElem returns a codeElem based on tok and node, or an error is returned.
  56. func newCodeElem(tok token.Token, node ast.Node) (codeElem, error) {
  57. var b strings.Builder
  58. err := format.Node(&b, token.NewFileSet(), node)
  59. if err != nil {
  60. return codeElem{}, err
  61. }
  62. return codeElem{tok, b.String()}, nil
  63. }
  64. // codeSet is a set of codeElems
  65. type codeSet struct {
  66. set map[codeElem]bool // true for all codeElems in the set
  67. }
  68. // newCodeSet returns a new codeSet
  69. func newCodeSet() *codeSet { return &codeSet{make(map[codeElem]bool)} }
  70. // add adds elem to c
  71. func (c *codeSet) add(elem codeElem) { c.set[elem] = true }
  72. // has returns true if elem is in c
  73. func (c *codeSet) has(elem codeElem) bool { return c.set[elem] }
  74. // isEmpty returns true if the set is empty
  75. func (c *codeSet) isEmpty() bool { return len(c.set) == 0 }
  76. // intersection returns a new set which is the intersection of c and a
  77. func (c *codeSet) intersection(a *codeSet) *codeSet {
  78. res := newCodeSet()
  79. for elem := range c.set {
  80. if a.has(elem) {
  81. res.add(elem)
  82. }
  83. }
  84. return res
  85. }
  86. // keepCommon is a filterFn for filtering the merged file with common declarations.
  87. func (c *codeSet) keepCommon(elem codeElem) bool {
  88. switch elem.tok {
  89. case token.VAR:
  90. // Remove all vars from the merged file
  91. return false
  92. case token.CONST, token.TYPE, token.FUNC, token.COMMENT:
  93. // Remove arch-specific consts, types, functions, and file-level comments from the merged file
  94. return c.has(elem)
  95. case token.IMPORT:
  96. // Keep imports, they are handled by filterImports
  97. return true
  98. }
  99. log.Fatalf("keepCommon: invalid elem %v", elem)
  100. return true
  101. }
  102. // keepArchSpecific is a filterFn for filtering the GOARC-specific files.
  103. func (c *codeSet) keepArchSpecific(elem codeElem) bool {
  104. switch elem.tok {
  105. case token.CONST, token.TYPE, token.FUNC:
  106. // Remove common consts, types, or functions from the arch-specific file
  107. return !c.has(elem)
  108. }
  109. return true
  110. }
  111. // srcFile represents a source file
  112. type srcFile struct {
  113. name string
  114. src []byte
  115. }
  116. // filterFn is a helper for filter
  117. type filterFn func(codeElem) bool
  118. // filter parses and filters Go source code from src, removing top
  119. // level declarations using keep as predicate.
  120. // For src parameter, please see docs for parser.ParseFile.
  121. func filter(src interface{}, keep filterFn) ([]byte, error) {
  122. // Parse the src into an ast
  123. fset := token.NewFileSet()
  124. f, err := parser.ParseFile(fset, "", src, parser.ParseComments)
  125. if err != nil {
  126. return nil, err
  127. }
  128. cmap := ast.NewCommentMap(fset, f, f.Comments)
  129. // Group const/type specs on adjacent lines
  130. var groups specGroups = make(map[string]int)
  131. var groupID int
  132. decls := f.Decls
  133. f.Decls = f.Decls[:0]
  134. for _, decl := range decls {
  135. switch decl := decl.(type) {
  136. case *ast.GenDecl:
  137. // Filter imports, consts, types, vars
  138. specs := decl.Specs
  139. decl.Specs = decl.Specs[:0]
  140. for i, spec := range specs {
  141. elem, err := newCodeElem(decl.Tok, spec)
  142. if err != nil {
  143. return nil, err
  144. }
  145. // Create new group if there are empty lines between this and the previous spec
  146. if i > 0 && fset.Position(specs[i-1].End()).Line < fset.Position(spec.Pos()).Line-1 {
  147. groupID++
  148. }
  149. // Check if we should keep this spec
  150. if keep(elem) {
  151. decl.Specs = append(decl.Specs, spec)
  152. groups.add(elem.src, groupID)
  153. }
  154. }
  155. // Check if we should keep this decl
  156. if len(decl.Specs) > 0 {
  157. f.Decls = append(f.Decls, decl)
  158. }
  159. case *ast.FuncDecl:
  160. // Filter funcs
  161. elem, err := newCodeElem(token.FUNC, decl)
  162. if err != nil {
  163. return nil, err
  164. }
  165. if keep(elem) {
  166. f.Decls = append(f.Decls, decl)
  167. }
  168. }
  169. }
  170. // Filter file level comments
  171. if cmap[f] != nil {
  172. commentGroups := cmap[f]
  173. cmap[f] = cmap[f][:0]
  174. for _, cGrp := range commentGroups {
  175. if keep(codeElem{token.COMMENT, cGrp.Text()}) {
  176. cmap[f] = append(cmap[f], cGrp)
  177. }
  178. }
  179. }
  180. f.Comments = cmap.Filter(f).Comments()
  181. // Generate code for the filtered ast
  182. var buf bytes.Buffer
  183. if err = format.Node(&buf, fset, f); err != nil {
  184. return nil, err
  185. }
  186. groupedSrc, err := groups.filterEmptyLines(&buf)
  187. if err != nil {
  188. return nil, err
  189. }
  190. return filterImports(groupedSrc)
  191. }
  192. // getCommonSet returns the set of consts, types, and funcs that are present in every file.
  193. func getCommonSet(files []srcFile) (*codeSet, error) {
  194. if len(files) == 0 {
  195. return nil, fmt.Errorf("no files provided")
  196. }
  197. // Use the first architecture file as the baseline
  198. baseSet, err := getCodeSet(files[0].src)
  199. if err != nil {
  200. return nil, err
  201. }
  202. // Compare baseline set with other architecture files: discard any element,
  203. // that doesn't exist in other architecture files.
  204. for _, f := range files[1:] {
  205. set, err := getCodeSet(f.src)
  206. if err != nil {
  207. return nil, err
  208. }
  209. baseSet = baseSet.intersection(set)
  210. }
  211. return baseSet, nil
  212. }
  213. // getCodeSet returns the set of all top-level consts, types, and funcs from src.
  214. // src must be string, []byte, or io.Reader (see go/parser.ParseFile docs)
  215. func getCodeSet(src interface{}) (*codeSet, error) {
  216. set := newCodeSet()
  217. fset := token.NewFileSet()
  218. f, err := parser.ParseFile(fset, "", src, parser.ParseComments)
  219. if err != nil {
  220. return nil, err
  221. }
  222. for _, decl := range f.Decls {
  223. switch decl := decl.(type) {
  224. case *ast.GenDecl:
  225. // Add const, and type declarations
  226. if !(decl.Tok == token.CONST || decl.Tok == token.TYPE) {
  227. break
  228. }
  229. for _, spec := range decl.Specs {
  230. elem, err := newCodeElem(decl.Tok, spec)
  231. if err != nil {
  232. return nil, err
  233. }
  234. set.add(elem)
  235. }
  236. case *ast.FuncDecl:
  237. // Add func declarations
  238. elem, err := newCodeElem(token.FUNC, decl)
  239. if err != nil {
  240. return nil, err
  241. }
  242. set.add(elem)
  243. }
  244. }
  245. // Add file level comments
  246. cmap := ast.NewCommentMap(fset, f, f.Comments)
  247. for _, cGrp := range cmap[f] {
  248. set.add(codeElem{token.COMMENT, cGrp.Text()})
  249. }
  250. return set, nil
  251. }
  252. // importName returns the identifier (PackageName) for an imported package
  253. func importName(iSpec *ast.ImportSpec) (string, error) {
  254. if iSpec.Name == nil {
  255. name, err := strconv.Unquote(iSpec.Path.Value)
  256. if err != nil {
  257. return "", err
  258. }
  259. return path.Base(name), nil
  260. }
  261. return iSpec.Name.Name, nil
  262. }
  263. // specGroups tracks grouped const/type specs with a map of line: groupID pairs
  264. type specGroups map[string]int
  265. // add spec source to group
  266. func (s specGroups) add(src string, groupID int) error {
  267. srcBytes, err := format.Source(bytes.TrimSpace([]byte(src)))
  268. if err != nil {
  269. return err
  270. }
  271. s[string(srcBytes)] = groupID
  272. return nil
  273. }
  274. // filterEmptyLines removes empty lines within groups of const/type specs.
  275. // Returns the filtered source.
  276. func (s specGroups) filterEmptyLines(src io.Reader) ([]byte, error) {
  277. scanner := bufio.NewScanner(src)
  278. var out bytes.Buffer
  279. var emptyLines bytes.Buffer
  280. prevGroupID := -1 // Initialize to invalid group
  281. for scanner.Scan() {
  282. line := bytes.TrimSpace(scanner.Bytes())
  283. if len(line) == 0 {
  284. fmt.Fprintf(&emptyLines, "%s\n", scanner.Bytes())
  285. continue
  286. }
  287. // Discard emptyLines if previous non-empty line belonged to the same
  288. // group as this line
  289. if src, err := format.Source(line); err == nil {
  290. groupID, ok := s[string(src)]
  291. if ok && groupID == prevGroupID {
  292. emptyLines.Reset()
  293. }
  294. prevGroupID = groupID
  295. }
  296. emptyLines.WriteTo(&out)
  297. fmt.Fprintf(&out, "%s\n", scanner.Bytes())
  298. }
  299. if err := scanner.Err(); err != nil {
  300. return nil, err
  301. }
  302. return out.Bytes(), nil
  303. }
  304. // filterImports removes unused imports from fileSrc, and returns a formatted src.
  305. func filterImports(fileSrc []byte) ([]byte, error) {
  306. fset := token.NewFileSet()
  307. file, err := parser.ParseFile(fset, "", fileSrc, parser.ParseComments)
  308. if err != nil {
  309. return nil, err
  310. }
  311. cmap := ast.NewCommentMap(fset, file, file.Comments)
  312. // create set of references to imported identifiers
  313. keepImport := make(map[string]bool)
  314. for _, u := range file.Unresolved {
  315. keepImport[u.Name] = true
  316. }
  317. // filter import declarations
  318. decls := file.Decls
  319. file.Decls = file.Decls[:0]
  320. for _, decl := range decls {
  321. importDecl, ok := decl.(*ast.GenDecl)
  322. // Keep non-import declarations
  323. if !ok || importDecl.Tok != token.IMPORT {
  324. file.Decls = append(file.Decls, decl)
  325. continue
  326. }
  327. // Filter the import specs
  328. specs := importDecl.Specs
  329. importDecl.Specs = importDecl.Specs[:0]
  330. for _, spec := range specs {
  331. iSpec := spec.(*ast.ImportSpec)
  332. name, err := importName(iSpec)
  333. if err != nil {
  334. return nil, err
  335. }
  336. if keepImport[name] {
  337. importDecl.Specs = append(importDecl.Specs, iSpec)
  338. }
  339. }
  340. if len(importDecl.Specs) > 0 {
  341. file.Decls = append(file.Decls, importDecl)
  342. }
  343. }
  344. // filter file.Imports
  345. imports := file.Imports
  346. file.Imports = file.Imports[:0]
  347. for _, spec := range imports {
  348. name, err := importName(spec)
  349. if err != nil {
  350. return nil, err
  351. }
  352. if keepImport[name] {
  353. file.Imports = append(file.Imports, spec)
  354. }
  355. }
  356. file.Comments = cmap.Filter(file).Comments()
  357. var buf bytes.Buffer
  358. err = format.Node(&buf, fset, file)
  359. if err != nil {
  360. return nil, err
  361. }
  362. return buf.Bytes(), nil
  363. }
  364. // merge extracts duplicate code from archFiles and merges it to mergeFile.
  365. // 1. Construct commonSet: the set of code that is idential in all archFiles.
  366. // 2. Write the code in commonSet to mergedFile.
  367. // 3. Remove the commonSet code from all archFiles.
  368. func merge(mergedFile string, archFiles ...string) error {
  369. // extract and validate the GOOS part of the merged filename
  370. goos, ok := getValidGOOS(mergedFile)
  371. if !ok {
  372. return fmt.Errorf("invalid GOOS in merged file name %s", mergedFile)
  373. }
  374. // Read architecture files
  375. var inSrc []srcFile
  376. for _, file := range archFiles {
  377. src, err := ioutil.ReadFile(file)
  378. if err != nil {
  379. return fmt.Errorf("cannot read archfile %s: %w", file, err)
  380. }
  381. inSrc = append(inSrc, srcFile{file, src})
  382. }
  383. // 1. Construct the set of top-level declarations common for all files
  384. commonSet, err := getCommonSet(inSrc)
  385. if err != nil {
  386. return err
  387. }
  388. if commonSet.isEmpty() {
  389. // No common code => do not modify any files
  390. return nil
  391. }
  392. // 2. Write the merged file
  393. mergedSrc, err := filter(inSrc[0].src, commonSet.keepCommon)
  394. if err != nil {
  395. return err
  396. }
  397. f, err := os.Create(mergedFile)
  398. if err != nil {
  399. return err
  400. }
  401. buf := bufio.NewWriter(f)
  402. fmt.Fprintln(buf, "// Code generated by mkmerge.go; DO NOT EDIT.")
  403. fmt.Fprintln(buf)
  404. fmt.Fprintf(buf, "// +build %s\n", goos)
  405. fmt.Fprintln(buf)
  406. buf.Write(mergedSrc)
  407. err = buf.Flush()
  408. if err != nil {
  409. return err
  410. }
  411. err = f.Close()
  412. if err != nil {
  413. return err
  414. }
  415. // 3. Remove duplicate declarations from the architecture files
  416. for _, inFile := range inSrc {
  417. src, err := filter(inFile.src, commonSet.keepArchSpecific)
  418. if err != nil {
  419. return err
  420. }
  421. err = ioutil.WriteFile(inFile.name, src, 0644)
  422. if err != nil {
  423. return err
  424. }
  425. }
  426. return nil
  427. }
  428. func main() {
  429. var mergedFile string
  430. flag.StringVar(&mergedFile, "out", "", "Write merged code to `FILE`")
  431. flag.Parse()
  432. // Expand wildcards
  433. var filenames []string
  434. for _, arg := range flag.Args() {
  435. matches, err := filepath.Glob(arg)
  436. if err != nil {
  437. fmt.Fprintf(os.Stderr, "Invalid command line argument %q: %v\n", arg, err)
  438. os.Exit(1)
  439. }
  440. filenames = append(filenames, matches...)
  441. }
  442. if len(filenames) < 2 {
  443. // No need to merge
  444. return
  445. }
  446. err := merge(mergedFile, filenames...)
  447. if err != nil {
  448. fmt.Fprintf(os.Stderr, "Merge failed with error: %v\n", err)
  449. os.Exit(1)
  450. }
  451. }