From 88058c3ce20bb88c2613b9b5ef9aa558bf8440b9 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Mon, 8 Jul 2019 12:36:40 +0200 Subject: hack/analyses -> dependencies/analyses Move the analyses scripts to the dependencies directory to avoid scattering of the dependency management. Signed-off-by: Valentin Rothberg --- dependencies/analyses/go-archive-analysis.sh | 12 ++++++++++++ dependencies/analyses/nm-symbols-analysis.sh | 15 +++++++++++++++ 2 files changed, 27 insertions(+) create mode 100755 dependencies/analyses/go-archive-analysis.sh create mode 100755 dependencies/analyses/nm-symbols-analysis.sh (limited to 'dependencies') diff --git a/dependencies/analyses/go-archive-analysis.sh b/dependencies/analyses/go-archive-analysis.sh new file mode 100755 index 000000000..f10145dad --- /dev/null +++ b/dependencies/analyses/go-archive-analysis.sh @@ -0,0 +1,12 @@ +#!/usr/bin/bash + +if [ -z "$WORK" ] +then + echo "WORK environment variable must be set" + exit 1 +fi + +grep --no-filename packagefile $WORK/**/importcfg \ + | awk '{ split($2, data, "="); printf "%s ", data[1]; system("du -sh " data[2]) }' \ + | awk '{ printf "%s %s\n", $2, $1 }' \ + | sort -u | sort -rh diff --git a/dependencies/analyses/nm-symbols-analysis.sh b/dependencies/analyses/nm-symbols-analysis.sh new file mode 100755 index 000000000..924246715 --- /dev/null +++ b/dependencies/analyses/nm-symbols-analysis.sh @@ -0,0 +1,15 @@ +#!/usr/bin/bash + +if test "$#" -ne 1; then + echo "invalid arguments: usage: $0 path/to/binary" + exit 1 +fi + +DATA=$(go tool nm -size "$1" \ + | awk 'NF==4 {printf "%s\t%s\t%s\n", $2, $3, $4}' \ + | grep -v -P "\t_\t" \ + | grep -P "\tt\t" \ + | awk ' {printf "%s\t\t%s\n", $1, $3} ' \ + ) + +echo "$DATA" -- cgit v1.2.3-54-g00ecf From fb31cc95a596b9c7c09cbc712acd60027c32ddfa Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Mon, 8 Jul 2019 14:04:32 +0200 Subject: analyses: add README.md Signed-off-by: Valentin Rothberg --- dependencies/analyses/README.md | 64 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 dependencies/analyses/README.md (limited to 'dependencies') diff --git a/dependencies/analyses/README.md b/dependencies/analyses/README.md new file mode 100644 index 000000000..70de3f621 --- /dev/null +++ b/dependencies/analyses/README.md @@ -0,0 +1,64 @@ +# A set of scripts and instructions that help to analyze and debloat go-lang dependencies. + +## Size of packages + +To analyze the size of all go packages used during the build process, pass the `-work -a` build flags to `go build`. +The `-a` flag forces go to rebuild all packages even if they are already up-to-date (e.g., in the build cache), while the `-work` flag instructs go to print path of the temporary work directory used for compiling the packages. +The path to the temporary work directory of `go-build` must be passed to `go-archive-analysis.sh` by setting it as an environment variable. +The analysis script will then read and parse the build data and print a sorted table of the package size in bytes followed by the package name. + +Running such an analysis on libpod may look as follows: + +``` +# 1) Build the podman binary with `-work -a`. + +[libpod]$ BUILDFLAGS="-work -a" make podman +[...] +WORK=/tmp/go-build794287815 + +# 2) Set the work directory as an environment variable and call the analysis script +[libpod]$ WORK=/tmp/go-build794287815 ./dependencies/analyses/go-archive-analysis.sh | head -n10 +17M github.com/containers/libpod/cmd/podman/cliconfig +13M github.com/containers/libpod/vendor/github.com/DataDog/zstd +10M github.com/containers/libpod/vendor/k8s.io/api/core/v1 +3.7M net/http +3.7M github.com/containers/libpod/libpod +3.2M runtime +2.7M github.com/containers/libpod/vendor/github.com/gogo/protobuf/proto +2.5M github.com/containers/libpod/vendor/k8s.io/apimachinery/pkg/apis/meta/v1 +2.3M github.com/containers/libpod/vendor/github.com/vishvananda/netlink +2.1M github.com/containers/libpod/cmd/podman/varlink +``` + +The output of the `go-archive-analysis.sh` script is a sorted table with the size in bytes followed by the package. +The size denotes the size of the compiled package (i.e., the `.a` file). + + +## Size of symbols in binary + +Once the binary is compiled, we can run another set of analyses on it. +The `nm-symbols-analysis.sh` is a wrapper around `go tool nm` and prints a table with size in bytes followed by the symbols name. +To avoid information overload, the scripts prints only symbols from the text/code segment. + +Running such an analysis on libpod may look as follows: + +``` +# 1) Compile the binary +[libpod]$ make podman +[...] + +# 2) Run the script with the binary as an argument +[libpod]$ ./dependencies/nm-symbols-analysis.sh ./bin/podman | grep "containers/libpod/libpod" | head -n10 +299 github.com/containers/libpod/libpod.(*BoltState).AddContainer +658 github.com/containers/libpod/libpod.(*BoltState).AddContainerToPod +2120 github.com/containers/libpod/libpod.(*BoltState).AddPod +3773 github.com/containers/libpod/libpod.(*BoltState).AddPod.func1 +965 github.com/containers/libpod/libpod.(*BoltState).AddVolume +1651 github.com/containers/libpod/libpod.(*BoltState).AddVolume.func1 +558 github.com/containers/libpod/libpod.(*BoltState).AllContainers +282 github.com/containers/libpod/libpod.(*BoltState).AllContainers.func1 +1121 github.com/containers/libpod/libpod.(*BoltState).AllContainers.func1.1 +558 github.com/containers/libpod/libpod.(*BoltState).AllPods +``` + +Running the script can help identify sources of bloat and reveal potential candidates (e.g., entire packages, types, or function) for refactoring. -- cgit v1.2.3-54-g00ecf From 849e2f3d4e1975bf02cfc7d2b32534294780a53b Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Mon, 8 Jul 2019 15:28:39 +0200 Subject: analyses: add dependency-tree.sh Add a new analysis script to print the dependency tree. Signed-off-by: Valentin Rothberg --- dependencies/analyses/README.md | 23 +++++++++++++++++++++++ dependencies/analyses/dependency-tree.sh | 13 +++++++++++++ 2 files changed, 36 insertions(+) create mode 100755 dependencies/analyses/dependency-tree.sh (limited to 'dependencies') diff --git a/dependencies/analyses/README.md b/dependencies/analyses/README.md index 70de3f621..2c6e4ca2e 100644 --- a/dependencies/analyses/README.md +++ b/dependencies/analyses/README.md @@ -1,5 +1,8 @@ # A set of scripts and instructions that help to analyze and debloat go-lang dependencies. +Note that all scripts mentioned below follow the [KISS principle](https://en.wikipedia.org/wiki/KISS_principle) on purpose. +The scripts are meant to be used in combination to aid in undestanding the packages' dependencies and how they contribute to the size of the compiled binary. + ## Size of packages To analyze the size of all go packages used during the build process, pass the `-work -a` build flags to `go build`. @@ -62,3 +65,23 @@ Running such an analysis on libpod may look as follows: ``` Running the script can help identify sources of bloat and reveal potential candidates (e.g., entire packages, types, or function) for refactoring. + + +## Dependency Tree + +Use the `dependency-tree.sh` script to figure out which package including which packages. +The output of the script has the format `package: dependency_1, dependency_2, ...`. +Each line is followed by a blank link to make it easier to read. +Note that the list of dependencies includes only the direct dependencies and not all transitive dependencies. +The transitive dependencies of a given package can be examined by running `go list -f '{{ .Name }}: {{ join .Deps ", " }}' $PACKAGE` or by browsing through the output of `dependency-tree.sh`. + +Running such a dependency-tree analysis may look as follows: + + +``` +[libpod]$ ./dependencies/analyses/dependency-tree.sh github.com/containers/libpod > tree.txt +[libpod]$ grep "^github.com/containers/libpod/pkg/registries" tree.txt +github.com/containers/libpod/pkg/registries: github.com/containers/libpod/vendor/github.com/containers/image/pkg/sysregistriesv2, github.com/containers/libpod/vendor/github.com/containers/image/types, github.com/containers/libpod/pkg/rootless, github.com/containers/libpod/vendor/github.com/docker/distribution/reference, github.com/containers/libpod/vendor/github.com/pkg/errors, os, path/filepath, strings +``` + +As shown above, the script's output can then be used to query for specific packages. diff --git a/dependencies/analyses/dependency-tree.sh b/dependencies/analyses/dependency-tree.sh new file mode 100755 index 000000000..3c9dccc51 --- /dev/null +++ b/dependencies/analyses/dependency-tree.sh @@ -0,0 +1,13 @@ +#!/usr/bin/bash + +if test "$#" -ne 1; then + echo "invalid arguments: usage: $0 path to package" + exit 1 +fi + +DATA=$(go list $1/... \ + | xargs -d '\n' go list -f '{{ .ImportPath }}: {{ join .Imports ", " }}' \ + | awk '{ printf "%s\n\n", $0 }' \ + ) + +echo "$DATA" -- cgit v1.2.3-54-g00ecf From 525c1ba89744d057cc558426ca53c49c9bb7cce5 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Tue, 9 Jul 2019 09:21:01 +0200 Subject: analyses: README: fix typos Signed-off-by: Valentin Rothberg --- dependencies/analyses/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'dependencies') diff --git a/dependencies/analyses/README.md b/dependencies/analyses/README.md index 2c6e4ca2e..dad925b47 100644 --- a/dependencies/analyses/README.md +++ b/dependencies/analyses/README.md @@ -1,12 +1,12 @@ -# A set of scripts and instructions that help to analyze and debloat go-lang dependencies. +# A set of scripts and instructions that help to analyze and debloat go-lang dependencies Note that all scripts mentioned below follow the [KISS principle](https://en.wikipedia.org/wiki/KISS_principle) on purpose. -The scripts are meant to be used in combination to aid in undestanding the packages' dependencies and how they contribute to the size of the compiled binary. +The scripts are meant to be used in combination to aid in understanding the packages' dependencies and how they contribute to the size of the compiled binary. ## Size of packages To analyze the size of all go packages used during the build process, pass the `-work -a` build flags to `go build`. -The `-a` flag forces go to rebuild all packages even if they are already up-to-date (e.g., in the build cache), while the `-work` flag instructs go to print path of the temporary work directory used for compiling the packages. +The `-a` flag forces go to rebuild all packages even if they are already up-to-date (e.g., in the build cache), while the `-work` flag instructs go to print the temporary work directory used for compiling the packages. The path to the temporary work directory of `go-build` must be passed to `go-archive-analysis.sh` by setting it as an environment variable. The analysis script will then read and parse the build data and print a sorted table of the package size in bytes followed by the package name. @@ -40,7 +40,7 @@ The size denotes the size of the compiled package (i.e., the `.a` file). ## Size of symbols in binary Once the binary is compiled, we can run another set of analyses on it. -The `nm-symbols-analysis.sh` is a wrapper around `go tool nm` and prints a table with size in bytes followed by the symbols name. +The `nm-symbols-analysis.sh` is a wrapper around `go tool nm` and prints a table with the size in bytes followed by the symbol's name. To avoid information overload, the scripts prints only symbols from the text/code segment. Running such an analysis on libpod may look as follows: @@ -69,9 +69,9 @@ Running the script can help identify sources of bloat and reveal potential candi ## Dependency Tree -Use the `dependency-tree.sh` script to figure out which package including which packages. +Use the `dependency-tree.sh` script to figure out which package includes which packages. The output of the script has the format `package: dependency_1, dependency_2, ...`. -Each line is followed by a blank link to make it easier to read. +Each line is followed by a blank line to make it easier to read. Note that the list of dependencies includes only the direct dependencies and not all transitive dependencies. The transitive dependencies of a given package can be examined by running `go list -f '{{ .Name }}: {{ join .Deps ", " }}' $PACKAGE` or by browsing through the output of `dependency-tree.sh`. @@ -84,4 +84,4 @@ Running such a dependency-tree analysis may look as follows: github.com/containers/libpod/pkg/registries: github.com/containers/libpod/vendor/github.com/containers/image/pkg/sysregistriesv2, github.com/containers/libpod/vendor/github.com/containers/image/types, github.com/containers/libpod/pkg/rootless, github.com/containers/libpod/vendor/github.com/docker/distribution/reference, github.com/containers/libpod/vendor/github.com/pkg/errors, os, path/filepath, strings ``` -As shown above, the script's output can then be used to query for specific packages. +As shown above, the script's output can then be used to query for specific packages (e.g, with `grep`). -- cgit v1.2.3-54-g00ecf From 9ae3e7c1eced61253330c2c8bd93d51ee97a6774 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Fri, 12 Jul 2019 11:19:08 +0200 Subject: analyses: README: consistent code examples Remove a blank line to make code examples more consistent and fix the path of the 2nd example. Signed-off-by: Valentin Rothberg --- dependencies/analyses/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'dependencies') diff --git a/dependencies/analyses/README.md b/dependencies/analyses/README.md index dad925b47..e05bc3e8f 100644 --- a/dependencies/analyses/README.md +++ b/dependencies/analyses/README.md @@ -14,7 +14,6 @@ Running such an analysis on libpod may look as follows: ``` # 1) Build the podman binary with `-work -a`. - [libpod]$ BUILDFLAGS="-work -a" make podman [...] WORK=/tmp/go-build794287815 @@ -51,7 +50,7 @@ Running such an analysis on libpod may look as follows: [...] # 2) Run the script with the binary as an argument -[libpod]$ ./dependencies/nm-symbols-analysis.sh ./bin/podman | grep "containers/libpod/libpod" | head -n10 +[libpod]$ ./dependencies/analyses/nm-symbols-analysis.sh ./bin/podman | grep "containers/libpod/libpod" | head -n10 299 github.com/containers/libpod/libpod.(*BoltState).AddContainer 658 github.com/containers/libpod/libpod.(*BoltState).AddContainerToPod 2120 github.com/containers/libpod/libpod.(*BoltState).AddPod -- cgit v1.2.3-54-g00ecf From 25d63f009dfb0bd646b761591a08989cfd9d4fda Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Fri, 12 Jul 2019 11:35:48 +0200 Subject: dependency-tree analysis: direct and transitive Change the script to generate two files. One including direct dependencies, the other including direct and transitive dependencies. Signed-off-by: Valentin Rothberg --- dependencies/analyses/README.md | 10 ++++++---- dependencies/analyses/dependency-tree.sh | 9 ++++++++- 2 files changed, 14 insertions(+), 5 deletions(-) (limited to 'dependencies') diff --git a/dependencies/analyses/README.md b/dependencies/analyses/README.md index e05bc3e8f..a440a0ebd 100644 --- a/dependencies/analyses/README.md +++ b/dependencies/analyses/README.md @@ -71,15 +71,17 @@ Running the script can help identify sources of bloat and reveal potential candi Use the `dependency-tree.sh` script to figure out which package includes which packages. The output of the script has the format `package: dependency_1, dependency_2, ...`. Each line is followed by a blank line to make it easier to read. -Note that the list of dependencies includes only the direct dependencies and not all transitive dependencies. -The transitive dependencies of a given package can be examined by running `go list -f '{{ .Name }}: {{ join .Deps ", " }}' $PACKAGE` or by browsing through the output of `dependency-tree.sh`. +The script generates two files: + + - `direct-tree.txt` - listing direct dependencies + - `transitive-tree.txt` - listing direct and transitive dependencies Running such a dependency-tree analysis may look as follows: ``` -[libpod]$ ./dependencies/analyses/dependency-tree.sh github.com/containers/libpod > tree.txt -[libpod]$ grep "^github.com/containers/libpod/pkg/registries" tree.txt +[libpod]$ ./dependencies/analyses/dependency-tree.sh github.com/containers/libpod +[libpod]$ grep "^github.com/containers/libpod/pkg/registries" direct-tree.txt github.com/containers/libpod/pkg/registries: github.com/containers/libpod/vendor/github.com/containers/image/pkg/sysregistriesv2, github.com/containers/libpod/vendor/github.com/containers/image/types, github.com/containers/libpod/pkg/rootless, github.com/containers/libpod/vendor/github.com/docker/distribution/reference, github.com/containers/libpod/vendor/github.com/pkg/errors, os, path/filepath, strings ``` diff --git a/dependencies/analyses/dependency-tree.sh b/dependencies/analyses/dependency-tree.sh index 3c9dccc51..9a2e3282d 100755 --- a/dependencies/analyses/dependency-tree.sh +++ b/dependencies/analyses/dependency-tree.sh @@ -10,4 +10,11 @@ DATA=$(go list $1/... \ | awk '{ printf "%s\n\n", $0 }' \ ) -echo "$DATA" +echo "$DATA" > direct-tree.txt + +DATA=$(go list $1/... \ + | xargs -d '\n' go list -f '{{ .ImportPath }}: {{ join .Deps ", " }}' \ + | awk '{ printf "%s\n\n", $0 }' \ + ) + +echo "$DATA" > transitive-tree.txt -- cgit v1.2.3-54-g00ecf From 294ddc472047f5a13bcf1d9b41407e18b1fad984 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Tue, 16 Jul 2019 14:14:48 +0200 Subject: dependency/analyses: simplify scripts Credits to bash wizard @edsantiago for the changes. Signed-off-by: Valentin Rothberg --- dependencies/analyses/dependency-tree.sh | 11 ++++------- dependencies/analyses/nm-symbols-analysis.sh | 10 ++-------- 2 files changed, 6 insertions(+), 15 deletions(-) (limited to 'dependencies') diff --git a/dependencies/analyses/dependency-tree.sh b/dependencies/analyses/dependency-tree.sh index 9a2e3282d..84085a50d 100755 --- a/dependencies/analyses/dependency-tree.sh +++ b/dependencies/analyses/dependency-tree.sh @@ -5,16 +5,13 @@ if test "$#" -ne 1; then exit 1 fi -DATA=$(go list $1/... \ +go list $1/... \ | xargs -d '\n' go list -f '{{ .ImportPath }}: {{ join .Imports ", " }}' \ | awk '{ printf "%s\n\n", $0 }' \ - ) + > direct-tree.tmp.$$ && mv -f direct-tree.tmp.$$ direct-tree.txt -echo "$DATA" > direct-tree.txt -DATA=$(go list $1/... \ +go list $1/... \ | xargs -d '\n' go list -f '{{ .ImportPath }}: {{ join .Deps ", " }}' \ | awk '{ printf "%s\n\n", $0 }' \ - ) - -echo "$DATA" > transitive-tree.txt + > transitive-tree.tmp.$$ && mv -f transitive-tree.tmp.$$ transitive-tree.txt diff --git a/dependencies/analyses/nm-symbols-analysis.sh b/dependencies/analyses/nm-symbols-analysis.sh index 924246715..361b746e4 100755 --- a/dependencies/analyses/nm-symbols-analysis.sh +++ b/dependencies/analyses/nm-symbols-analysis.sh @@ -5,11 +5,5 @@ if test "$#" -ne 1; then exit 1 fi -DATA=$(go tool nm -size "$1" \ - | awk 'NF==4 {printf "%s\t%s\t%s\n", $2, $3, $4}' \ - | grep -v -P "\t_\t" \ - | grep -P "\tt\t" \ - | awk ' {printf "%s\t\t%s\n", $1, $3} ' \ - ) - -echo "$DATA" +go tool nm -size "$1" \ + | awk 'NF==4 && $3=="t" {printf "%s\t\t%s\n", $2, $4}' -- cgit v1.2.3-54-g00ecf