Clair の仕組みメモ

func (r *result) Do(ctx context.Context, s indexer.VersionedScanner, l *claircore.Layer) error {
	var err error
	switch s := s.(type) {
	case indexer.PackageScanner:
		r.pkgs, err = s.Scan(ctx, l)
	case indexer.DistributionScanner:
		r.dists, err = s.Scan(ctx, l)
	case indexer.RepositoryScanner:
		r.repos, err = s.Scan(ctx, l)
	default:
		panic(fmt.Sprintf("programmer error: unknown type %T used as scanner", s))
	}

const osReleasePath = `etc/os-release`

func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Distribution, error) {
    (snip)
	files, err := l.Files(osReleasePath)
	if err != nil {
		zlog.Debug(ctx).Msg("didn't find an os-release")
		return nil, nil
	}
	for _, buff := range files {
		dist := ds.parse(buff)
		if dist != nil {
			return []*claircore.Distribution{dist}, nil
		}
	}

func (l *Layer) Files(paths ...string) (map[string]*bytes.Buffer, error) {
	r, err := l.Reader()
	if err != nil {
		return nil, err
	}
	defer r.Close()
	rs := r.(io.ReadSeeker)
    (snip)
	for rs.Seek(0, io.SeekStart); again; rs.Seek(0, io.SeekStart) {
		again = false
		tr := tar.NewReader(rs)
		hdr, err := tr.Next()
		for ; err == nil; hdr, err = tr.Next() {
			name := filepath.Clean(hdr.Name)
			// check if the current header has a path name we are
			// searching for.
			if _, ok := want[name]; !ok {
				continue
			}
			delete(want, name)

			switch hdr.Typeflag {
                (snip)

			case tar.TypeReg:
				b := make([]byte, hdr.Size)
				if n, err := io.ReadFull(tr, b); int64(n) != hdr.Size || err != nil {
					return nil, fmt.Errorf("claircore: unable to read file from archive: read %d bytes (wanted: %d): %w", n, hdr.Size, err)
				}
				f[name] = bytes.NewBuffer(b)

    (snip)

	// If there's nothing in the "f" map, we didn't find anything.
	if len(f) == 0 {
		return nil, ErrNotFound
	}
	return f, nil

var awsRegexes = []awsRegex{
	{
		release: Linux1,
		regexp:  regexp.MustCompile(`Amazon Linux AMI 2018.03`),
	},
	{
		release: Linux2,
		regexp:  regexp.MustCompile(`Amazon Linux 2`),
	},
}

// DbNames is a set of files that make up an rpm database.
var dbnames = map[string]struct{}{
	"Packages": {},
}

func (ps *Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*claircore.Package, error) {
    (snip)

	possible := make(map[string]int)
	tr := tar.NewReader(rd)
	// Find possible rpm dbs
	// If none found, return
	var h *tar.Header
	for h, err = tr.Next(); err == nil; h, err = tr.Next() {
		n := filepath.Base(h.Name)
		d := filepath.Dir(h.Name)
		if _, ok := dbnames[n]; ok && checkMagic(ctx, tr) {
			possible[d]++
		}
	}

	(snip)

	root, err := ioutil.TempDir("", "rpmscanner.")

	(snip)
	for h, err = tr.Next(); err == nil; h, err = tr.Next() {
		if strings.HasPrefix(filepath.Base(h.Name), ".wh.") {
			// Whiteout, skip.
			stats.Whiteout++
			continue
		}

		(snip)

		// Build the path on the filesystem.
		tgt := relPath(root, h.Name)
		// Since tar, as a format, doesn't impose ordering requirements, make
		// sure to create all parent directories of the current entry.
		d := filepath.Dir(tgt)

		(snip)

		// Populate the target file.
		var err error
		switch h.Typeflag {
		case tar.TypeDir:
			m := h.FileInfo().Mode() | dirMode
            (snip)
			err = os.Mkdir(tgt, m)
			// Make sure to preempt the MkdirAll call if the entries were
			// ordered nicely.
			made[d] = struct{}{}
			stats.Dir++
		case tar.TypeReg:
			m := h.FileInfo().Mode() | fileMode
			var f *os.File
			f, err = os.OpenFile(tgt, os.O_CREATE|os.O_WRONLY, m)

			if err != nil {
				break // Handle after the switch.
			}
			_, err = io.Copy(f, tr)
			if err := f.Close(); err != nil {
				zlog.Warn(ctx).Err(err).Msg("error closing new file")
			}
			stats.Reg++
		case tar.TypeSymlink:
			// Normalize the link target into the root.
			ln := relPath(root, h.Linkname)
			err = os.Symlink(ln, tgt)
			stats.Symlink++
		case tar.TypeLink:
			// Normalize the link target into the root.
			ln := relPath(root, h.Linkname)
			_, exists := os.Lstat(ln)
			switch {
			case errors.Is(exists, nil):
				err = os.Link(ln, tgt)
			case errors.Is(exists, os.ErrNotExist):
				// Push onto a queue to fix later. Link(2) is documented to need
				// a valid target, unlike symlink(2), which allows a missing
				// target. Combined with tar's lack of ordering, this seems like
				// the best solution.
				deferLn = append(deferLn, [2]string{ln, tgt})
			default:
				err = exists
			}
			stats.Link++
		default:

for _, db := range found {
		zlog.Debug(ctx).Str("db", db).Msg("examining database")

		cmd := exec.CommandContext(ctx, "rpm",
			`--root`, root, `--dbpath`, db,
			`--query`, `--all`, `--queryformat`, queryFmt)
		r, err := cmd.StdoutPipe()

var (
    (snip)
	Repository = claircore.Repository{
		Name: "pypi",
		URI:  "https://pypi.org/simple",
	}
)

func (rs *RepoScanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*claircore.Repository, error) {
    for h, err = tr.Next(); err == nil; h, err = tr.Next() {
		n, err := filepath.Rel("/", filepath.Join("/", h.Name))
		if err != nil {
			return nil, err
		}
		switch {
		case h.Typeflag != tar.TypeReg:
			// Should we chase symlinks with the correct name?
			continue
		case strings.HasSuffix(n, `.egg-info/PKG-INFO`):
			zlog.Debug(ctx).Str("file", n).Msg("found egg")
		case strings.HasSuffix(n, `.dist-info/METADATA`):
			zlog.Debug(ctx).Str("file", n).Msg("found wheel")
		default:
			continue
		}

		// Just claim these came from pypi.
		return []*claircore.Repository{&Repository}, nil
	}

func (ps *Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*claircore.Package, error) {
    (snip)
    for h, err = tr.Next(); err == nil; h, err = tr.Next() {
		n, err := filepath.Rel("/", filepath.Join("/", h.Name))
		if err != nil {
			return nil, err
		}
		switch {
		case h.Typeflag != tar.TypeReg:
			// Should we chase symlinks with the correct name?
			continue
		case strings.HasSuffix(n, `.egg-info/PKG-INFO`):
			zlog.Debug(ctx).Str("file", n).Msg("found egg")
		case strings.HasSuffix(n, `.dist-info/METADATA`):
			zlog.Debug(ctx).Str("file", n).Msg("found wheel")
(snip)

// NewEcosystem provides the set of scanners and coalescers for the rpm ecosystem
func NewEcosystem(ctx context.Context) *indexer.Ecosystem {
	return &indexer.Ecosystem{
		PackageScanners: func(ctx context.Context) ([]indexer.PackageScanner, error) {
			return []indexer.PackageScanner{&Scanner{}}, nil
		},
		DistributionScanners: func(ctx context.Context) ([]indexer.DistributionScanner, error) {
			return []indexer.DistributionScanner{
				&aws.DistributionScanner{},
				&oracle.DistributionScanner{},
				&suse.DistributionScanner{},
				&photon.DistributionScanner{},
			}, nil
		},
		RepositoryScanners: func(ctx context.Context) ([]indexer.RepositoryScanner, error) {
			return []indexer.RepositoryScanner{}, nil
		},
		Coalescer: func(ctx context.Context) (indexer.Coalescer, error) {
			return linux.NewCoalescer(), nil
		},
	}
}

var scanners = []indexer.PackageScanner{&Scanner{}}
var reposcanners = []indexer.RepositoryScanner{&RepoScanner{}}

// NewEcosystem provides the set of scanners for the python ecosystem.
func NewEcosystem(ctx context.Context) *indexer.Ecosystem {
	return &indexer.Ecosystem{
		PackageScanners:      func(_ context.Context) ([]indexer.PackageScanner, error) { return scanners, nil },
		DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) { return nil, nil },
		RepositoryScanners:   func(_ context.Context) ([]indexer.RepositoryScanner, error) { return reposcanners, nil },
		Coalescer:            NewCoalescer,
	}
}

var requiredFiles = []string{
	"usr/lib/sysimage/rpm/Packages",
	"var/lib/rpm/Packages",
}

func (a rpmPkgAnalyzer) Analyze(_ context.Context, target analyzer.AnalysisTarget) (*analyzer.AnalysisResult, error) {
	parsedPkgs, installedFiles, err := a.parsePkgInfo(target.Content)
	if err != nil {
		return nil, xerrors.Errorf("failed to parse rpmdb: %w", err)
	}
	(snip)

func (a rpmPkgAnalyzer) parsePkgInfo(packageBytes []byte) ([]types.Package, []string, error) {
	tmpDir, err := ioutil.TempDir("", "rpm")
	defer os.RemoveAll(tmpDir)
	if err != nil {
		return nil, nil, xerrors.Errorf("failed to create a temp dir: %w", err)
	}

	filename := filepath.Join(tmpDir, "Packages")
	err = ioutil.WriteFile(filename, packageBytes, 0700)
	if err != nil {
		return nil, nil, xerrors.Errorf("failed to write a package file: %w", err)
	}

	// rpm-python 4.11.3 rpm-4.11.3-35.el7.src.rpm
	// Extract binary package names because RHSA refers to binary package names.
	db, err := rpmdb.Open(filename)
	if err != nil {
		return nil, nil, xerrors.Errorf("failed to open RPM DB: %w", err)
	}

	// equivalent:
	//   new version: rpm -qa --qf "%{NAME} %{EPOCHNUM} %{VERSION} %{RELEASE} %{SOURCERPM} %{ARCH}\n"
	//   old version: rpm -qa --qf "%{NAME} %{EPOCH} %{VERSION} %{RELEASE} %{SOURCERPM} %{ARCH}\n"
	pkgList, err := db.ListPackages()
	if err != nil {
	(snip)

func (a rpmPkgAnalyzer) Required(filePath string, _ os.FileInfo) bool {
	return utils.StringInSlice(filePath, requiredFiles)
}

func (a Analyzer) AnalyzeFile(ctx context.Context, wg *sync.WaitGroup, limit *semaphore.Weighted, result *AnalysisResult,
	dir, filePath string, info os.FileInfo, opener Opener) error {
	if info.IsDir() {
		return nil
	}
	for _, d := range a.drivers {
		// filepath extracted from tar file doesn't have the prefix "/"
		if !d.Required(strings.TrimLeft(filePath, "/"), info) {
			continue
		}
		b, err := opener()
		if err != nil {
			return xerrors.Errorf("unable to open a file (%s): %w", filePath, err)
		}
		(snip)

		go func(a analyzer, target AnalysisTarget) {
			defer limit.Release(1)
			defer wg.Done()

			ret, err := a.Analyze(ctx, target)
			if err != nil && !xerrors.Is(err, aos.AnalyzeOSError) {
				log.Logger.Debugf("Analysis error: %s", err)
				return
			}
			result.Merge(ret)
		}(d, AnalysisTarget{Dir: dir, FilePath: filePath, Content: b})
	}
	return nil
}

func (a Artifact) inspectLayer(ctx context.Context, diffID string) (types.BlobInfo, error) {
    (snip)
	opqDirs, whFiles, err := a.walker.Walk(r, func(filePath string, info os.FileInfo, opener analyzer.Opener) error {
		if err = a.analyzer.AnalyzeFile(ctx, &wg, limit, result, "", filePath, info, opener); err != nil {
			return xerrors.Errorf("failed to analyze %s: %w", filePath, err)
		}
		return nil
	})
    (snip)

func (w LayerTar) Walk(layer io.Reader, analyzeFn WalkFunc) ([]string, []string, error) {
	var opqDirs, whFiles, skipDirs []string
	tr := tar.NewReader(layer)
    (snip)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
        (snip)

		switch hdr.Typeflag {
		case tar.TypeDir:
			if w.shouldSkipDir(filePath) {
				skipDirs = append(skipDirs, filePath)
				continue
			}

        (snip)
		// A symbolic/hard link or regular file will reach here.
		err = analyzeFn(filePath, hdr.FileInfo(), w.fileOnceOpener(tr))
		if err != nil {
			return nil, nil, xerrors.Errorf("failed to analyze file: %w", err)
		}
	}
	return opqDirs, whFiles, nil
}

// fileOnceOpener opens a file once and the content is shared so that some analyzers can use the same data
func (w *walker) fileOnceOpener(r io.Reader) func() ([]byte, error) {
	var once sync.Once
	var b []byte
	var err error

	return func() ([]byte, error) {
		once.Do(func() {
			b, err = io.ReadAll(r)
		})
		if err != nil {
			return nil, xerrors.Errorf("unable to read the file: %w", err)
		}
		return b, nil
	}
}

func (r *AnalysisResult) Merge(new *AnalysisResult) {
(snip)
	if len(new.PackageInfos) > 0 {
		r.PackageInfos = append(r.PackageInfos, new.PackageInfos...)
	}

	if len(new.Applications) > 0 {
		r.Applications = append(r.Applications, new.Applications...)
	}
(snip)

func ApplyLayers(layers []types.BlobInfo) types.ArtifactDetail {
	sep := "/"
	nestedMap := nested.Nested{}
	var mergedLayer types.ArtifactDetail

	for _, layer := range layers {
		for _, opqDir := range layer.OpaqueDirs {
			opqDir = strings.TrimSuffix(opqDir, sep) //this is necessary so that an empty element is not contribute into the array of the DeleteByString function
			_ = nestedMap.DeleteByString(opqDir, sep)
		}
		for _, whFile := range layer.WhiteoutFiles {
			_ = nestedMap.DeleteByString(whFile, sep)
		}

		if layer.OS != nil {
			mergedLayer.OS = layer.OS
		}

		if layer.Repository != nil {
			mergedLayer.Repository = layer.Repository
		}

		for _, pkgInfo := range layer.PackageInfos {
			key := fmt.Sprintf("%s/type:ospkg", pkgInfo.FilePath)
			nestedMap.SetByString(key, sep, pkgInfo)
		}
		for _, app := range layer.Applications {
			key := fmt.Sprintf("%s/type:%s", app.FilePath, app.Type)
			nestedMap.SetByString(key, sep, app)
		}
(snip)
	}

	_ = nestedMap.Walk(func(keys []string, value interface{}) error {
		switch v := value.(type) {
		case types.PackageInfo:
			mergedLayer.Packages = append(mergedLayer.Packages, v.Packages...)
		case types.Application:
			mergedLayer.Applications = append(mergedLayer.Applications, v)
		case types.Misconfiguration:
			mergedLayer.Misconfigurations = append(mergedLayer.Misconfigurations, v)
		case types.Secret:
			mergedLayer.Secrets = append(mergedLayer.Secrets, v)
		case types.CustomResource:
			mergedLayer.CustomResources = append(mergedLayer.CustomResources, v)
		}
		return nil
	})

	for i, pkg := range mergedLayer.Packages {
		originLayerDigest, originLayerDiffID, buildInfo := lookupOriginLayerForPkg(pkg, layers)
		mergedLayer.Packages[i].Layer = types.Layer{
			Digest: originLayerDigest,
			DiffID: originLayerDiffID,
		}
		mergedLayer.Packages[i].BuildInfo = buildInfo
	}

(snip)

	// Aggregate python/ruby/node.js packages
	aggregate(&mergedLayer)

	return mergedLayer
}

// ArtifactDetail is generated by applying blobs
type ArtifactDetail struct {
	OS                *OS                `json:",omitempty"`
	Repository        *Repository        `json:",omitempty"`
	Packages          []Package          `json:",omitempty"`
	Applications      []Application      `json:",omitempty"`
	Misconfigurations []Misconfiguration `json:",omitempty"`
	Secrets           []Secret           `json:",omitempty"`

	// HistoryPackages are packages extracted from RUN instructions
	HistoryPackages []Package `json:",omitempty"`

	// CustomResources hold analysis results from custom analyzers.
	// It is for extensibility and not used in OSS.
	CustomResources []CustomResource `json:",omitempty"`
}