From c24b2595fa1b9e89902a6588afc87678b0fe4c1f Mon Sep 17 00:00:00 2001 From: jep Date: Tue, 27 Jul 2021 14:20:58 +0200 Subject: [PATCH 1/3] SSL certs flag --- cmd/amass/enum.go | 8 ++- config/config.go | 3 ++ doc/user_guide.md | 1 + enum/active.go | 33 ------------ enum/certs.go | 134 ++++++++++++++++++++++++++++++++++++++++++++++ enum/enum.go | 6 +++ 6 files changed, 151 insertions(+), 34 deletions(-) create mode 100644 enum/certs.go diff --git a/cmd/amass/enum.go b/cmd/amass/enum.go index 3981d44ab..b6105f508 100644 --- a/cmd/amass/enum.go +++ b/cmd/amass/enum.go @@ -73,6 +73,7 @@ type enumArgs struct { Silent bool Sources bool Verbose bool + Certs bool } Filepaths struct { AllFilePrefix string @@ -128,6 +129,7 @@ func defineEnumOptionFlags(enumFlags *flag.FlagSet, args *enumArgs) { enumFlags.BoolVar(&args.Options.Silent, "silent", false, "Disable all output during execution") enumFlags.BoolVar(&args.Options.Sources, "src", false, "Print data sources for the discovered names") enumFlags.BoolVar(&args.Options.Verbose, "v", false, "Output status / debug / troubleshooting info") + enumFlags.BoolVar(&args.Options.Certs, "c", false, "Determines if ssl certifications should be extracted") } func defineEnumFilepathFlags(enumFlags *flag.FlagSet, args *enumArgs) { @@ -653,7 +655,7 @@ func processEnumInputFiles(args *enumArgs) error { for _, f := range args.Filepaths.Resolvers { list, err := config.GetListFromFile(f) if err != nil { - return fmt.Errorf("Failed to parse the esolver file: %v", err) + return fmt.Errorf("Failed to parse the resolver file: %v", err) } args.Resolvers.InsertMany(list...) } @@ -710,6 +712,7 @@ func (e enumArgs) OverrideConfig(conf *config.Config) error { } if e.Options.Active { conf.Active = true + conf.Certs = true conf.Passive = false } if e.Options.Passive { @@ -730,6 +733,9 @@ func (e enumArgs) OverrideConfig(conf *config.Config) error { if e.MaxDNSQueries > 0 { conf.MaxDNSQueries = e.MaxDNSQueries } + if e.Options.Certs { + conf.Certs = true + } if len(e.Included) > 0 { conf.SourceFilter.Include = true diff --git a/config/config.go b/config/config.go index 19aed46f8..ba7f62110 100644 --- a/config/config.go +++ b/config/config.go @@ -121,6 +121,9 @@ type Config struct { // Determines if zone transfers will be attempted Active bool + // Determines if ssl certifications should be extracted + Certs bool + // A blacklist of subdomain names that will not be investigated Blacklist []string blacklistLock sync.Mutex diff --git a/doc/user_guide.md b/doc/user_guide.md index 75ab72860..edab40016 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -131,6 +131,7 @@ This subcommand will perform DNS enumeration and network mapping while populatin | -src | Print data sources for the discovered names | amass enum -src -d example.com | | -timeout | Number of minutes to execute the enumeration | amass enum -timeout 30 -d example.com | | -w | Path to a different wordlist file | amass enum -brute -w wordlist.txt -d example.com | +| -c | Determines if ssl certifications should be extracted(This is enabled by default on -active mode) | amass enum -c -d example.com ### The 'viz' Subcommand diff --git a/enum/active.go b/enum/active.go index 1fa51f5c6..fede58600 100644 --- a/enum/active.go +++ b/enum/active.go @@ -69,8 +69,6 @@ func (a *activeTask) Process(ctx context.Context, data pipeline.Data, tp pipelin switch data.(type) { case *requests.DNSRequest: ok = true - case *requests.AddrRequest: - ok = true case *requests.ZoneXFRRequest: ok = true } @@ -114,10 +112,6 @@ func (a *activeTask) processTask() { switch v := args.Data.(type) { case *requests.DNSRequest: go a.crawlName(args.Ctx, v, args.Params) - case *requests.AddrRequest: - if v.InScope { - go a.certEnumeration(args.Ctx, v, args.Params) - } case *requests.ZoneXFRRequest: go a.zoneTransfer(args.Ctx, v, args.Params) go a.zoneWalk(args.Ctx, v, args.Params) @@ -170,33 +164,6 @@ func (a *activeTask) crawlName(ctx context.Context, req *requests.DNSRequest, tp } } -func (a *activeTask) certEnumeration(ctx context.Context, req *requests.AddrRequest, tp pipeline.TaskParams) { - defer func() { a.tokenPool <- struct{}{} }() - - if req == nil || !req.Valid() { - return - } - - for _, name := range http.PullCertificateNames(ctx, req.Address, a.enum.Config.Ports) { - select { - case <-ctx.Done(): - return - default: - } - - if n := strings.TrimSpace(name); n != "" { - if domain := a.enum.Config.WhichDomain(n); domain != "" { - pipeline.SendData(ctx, "new", &requests.DNSRequest{ - Name: n, - Domain: domain, - Tag: requests.CERT, - Source: "Active Cert", - }, tp) - } - } - } -} - func (a *activeTask) zoneTransfer(ctx context.Context, req *requests.ZoneXFRRequest, tp pipeline.TaskParams) { select { case <-ctx.Done(): diff --git a/enum/certs.go b/enum/certs.go new file mode 100644 index 000000000..e26528a9b --- /dev/null +++ b/enum/certs.go @@ -0,0 +1,134 @@ +package enum + +import ( + "context" + "github.com/OWASP/Amass/v3/net/http" + "github.com/OWASP/Amass/v3/requests" + "github.com/caffix/pipeline" + "github.com/caffix/queue" + "strings" +) + +// certsTask is the task that handles all requests related to ssl extraction within the pipeline. +type certsTask struct { + enum *Enumeration + queue queue.Queue + tokenPool chan struct{} +} + +type certsTaskArgs struct { + Ctx context.Context + Data pipeline.Data + Params pipeline.TaskParams +} + +// newCertsTask returns a certsTask specific to the provided Enumeration. +func newCertsTask(e *Enumeration, max int) *certsTask { + if max <= 0 { + return nil + } + + tokenPool := make(chan struct{}, max) + for i := 0; i < max; i++ { + tokenPool <- struct{}{} + } + + a := &certsTask{ + enum: e, + queue: queue.NewQueue(), + tokenPool: tokenPool, + } + + go a.ProcessQueue() + return a +} + +func (c *certsTask) Stop() { + c.queue.Process(func(e interface{}) {}) +} + +func (c *certsTask) ProcessQueue() { + for { + select { + case <-c.enum.done: + return + case <-c.queue.Signal(): + c.processTask() + } + } +} + +// Process implements the pipeline Task interface. +func (c *certsTask) Process(ctx context.Context, data pipeline.Data, tp pipeline.TaskParams) (pipeline.Data, error) { + select { + case <-ctx.Done(): + return nil, nil + default: + } + + var ok bool + switch data.(type) { + case *requests.AddrRequest: + ok = true + } + + if ok { + c.queue.Append(&certsTaskArgs{ + Ctx: ctx, + Data: data.Clone(), + Params: tp, + }) + } + + return data, nil +} + +func (c *certsTask) processTask() { + select { + case <-c.enum.ctx.Done(): + return + case <-c.enum.done: + return + case <-c.tokenPool: + element, ok := c.queue.Next() + if !ok { + c.tokenPool <- struct{}{} + return + } + + args := element.(*certsTaskArgs) + switch v := args.Data.(type) { + case *requests.AddrRequest: + if v.InScope { + go c.certEnumeration(args.Ctx, v, args.Params) + } + } + } +} + +func (c *certsTask) certEnumeration(ctx context.Context, req *requests.AddrRequest, tp pipeline.TaskParams) { + defer func() { c.tokenPool <- struct{}{} }() + + if req == nil || !req.Valid() { + return + } + + for _, name := range http.PullCertificateNames(ctx, req.Address, c.enum.Config.Ports) { + select { + case <-ctx.Done(): + return + default: + } + + if n := strings.TrimSpace(name); n != "" { + if domain := c.enum.Config.WhichDomain(n); domain != "" { + pipeline.SendData(ctx, "new", &requests.DNSRequest{ + Name: n, + Domain: domain, + Tag: requests.CERT, + Source: "Active Cert", + }, tp) + } + } + } +} diff --git a/enum/enum.go b/enum/enum.go index f5e228336..fe266eb83 100644 --- a/enum/enum.go +++ b/enum/enum.go @@ -120,6 +120,12 @@ func (e *Enumeration) Start(ctx context.Context) error { stages = append(stages, pipeline.FIFO("active", activetask)) } + if e.Config.Certs { + certstask := newCertsTask(e, maxActivePipelineTasks) + defer certstask.Stop() + + stages = append(stages, pipeline.FIFO("certs", certstask)) + } /* * Now that the pipeline input source has been setup, names provided From 28e5ebb093f72e077315de26ae664d98ebd48525 Mon Sep 17 00:00:00 2001 From: jep Date: Tue, 27 Jul 2021 18:31:01 +0200 Subject: [PATCH 2/3] nocerts and noaxfr flags --- cmd/amass/enum.go | 30 ++++++++--- config/config.go | 9 ++-- doc/user_guide.md | 3 +- enum/active.go | 45 ++++++++++++++-- enum/certs.go | 134 ---------------------------------------------- enum/enum.go | 6 --- 6 files changed, 72 insertions(+), 155 deletions(-) delete mode 100644 enum/certs.go diff --git a/cmd/amass/enum.go b/cmd/amass/enum.go index b6105f508..917c4a183 100644 --- a/cmd/amass/enum.go +++ b/cmd/amass/enum.go @@ -73,7 +73,8 @@ type enumArgs struct { Silent bool Sources bool Verbose bool - Certs bool + NoCerts bool + NoAxfr bool } Filepaths struct { AllFilePrefix string @@ -129,7 +130,8 @@ func defineEnumOptionFlags(enumFlags *flag.FlagSet, args *enumArgs) { enumFlags.BoolVar(&args.Options.Silent, "silent", false, "Disable all output during execution") enumFlags.BoolVar(&args.Options.Sources, "src", false, "Print data sources for the discovered names") enumFlags.BoolVar(&args.Options.Verbose, "v", false, "Output status / debug / troubleshooting info") - enumFlags.BoolVar(&args.Options.Certs, "c", false, "Determines if ssl certifications should be extracted") + enumFlags.BoolVar(&args.Options.NoCerts, "nocerts", false, "Disables certificate name grabs when -active mode is enabled") + enumFlags.BoolVar(&args.Options.NoAxfr, "noaxfr", false, "Disables zone transfers when -active mode is enabled") } func defineEnumFilepathFlags(enumFlags *flag.FlagSet, args *enumArgs) { @@ -371,9 +373,19 @@ func argsAndConfig(clArgs []string) (*config.Config, *enumArgs) { r.Fprintln(color.Error, "IP addresses cannot be provided without DNS resolution") os.Exit(1) } - if !cfg.Active && len(args.Ports) > 0 { - r.Fprintln(color.Error, "Ports can only be scanned in the active mode") - os.Exit(1) + if !cfg.Active { + if len(args.Ports) > 0 { + r.Fprintln(color.Error, "Ports can only be scanned in the active mode") + os.Exit(1) + } + if args.Options.NoCerts { + r.Fprintln(color.Error, "Certificate name grabbing can only be disabled in the active mode") + os.Exit(1) + } + if args.Options.NoAxfr { + r.Fprintln(color.Error, "Zone transfers can only be disabled in the active mode") + os.Exit(1) + } } if len(cfg.Domains()) == 0 { r.Fprintln(color.Error, "Configuration error: No root domain names were provided") @@ -712,7 +724,6 @@ func (e enumArgs) OverrideConfig(conf *config.Config) error { } if e.Options.Active { conf.Active = true - conf.Certs = true conf.Passive = false } if e.Options.Passive { @@ -733,8 +744,11 @@ func (e enumArgs) OverrideConfig(conf *config.Config) error { if e.MaxDNSQueries > 0 { conf.MaxDNSQueries = e.MaxDNSQueries } - if e.Options.Certs { - conf.Certs = true + if e.Options.NoCerts { + conf.NoCerts = true + } + if e.Options.NoAxfr { + conf.NoAxfr = true } if len(e.Included) > 0 { diff --git a/config/config.go b/config/config.go index ba7f62110..5f4404412 100644 --- a/config/config.go +++ b/config/config.go @@ -118,11 +118,14 @@ type Config struct { // Only access the data sources for names and return results? Passive bool - // Determines if zone transfers will be attempted + // Determines if zone transfers and ssl certification extraction will be attempted Active bool - // Determines if ssl certifications should be extracted - Certs bool + // Determines if certificate name grabbing will be attempted + NoCerts bool + + // Determines if zone transfers will be attempted + NoAxfr bool // A blacklist of subdomain names that will not be investigated Blacklist []string diff --git a/doc/user_guide.md b/doc/user_guide.md index edab40016..b2af46ebb 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -131,7 +131,8 @@ This subcommand will perform DNS enumeration and network mapping while populatin | -src | Print data sources for the discovered names | amass enum -src -d example.com | | -timeout | Number of minutes to execute the enumeration | amass enum -timeout 30 -d example.com | | -w | Path to a different wordlist file | amass enum -brute -w wordlist.txt -d example.com | -| -c | Determines if ssl certifications should be extracted(This is enabled by default on -active mode) | amass enum -c -d example.com +| -nocerts | Disables certificate name grabs when -active mode is enabled | amass enum -active --no-certs -d example.com +| -noaxfr | Disables zone transfers when -active mode is enabled | amass enum -active -no-axfr -d example.com ### The 'viz' Subcommand diff --git a/enum/active.go b/enum/active.go index fede58600..c798d1631 100644 --- a/enum/active.go +++ b/enum/active.go @@ -69,8 +69,14 @@ func (a *activeTask) Process(ctx context.Context, data pipeline.Data, tp pipelin switch data.(type) { case *requests.DNSRequest: ok = true + case *requests.AddrRequest: + if !a.enum.Config.NoCerts { + ok = true + } case *requests.ZoneXFRRequest: - ok = true + if !a.enum.Config.NoAxfr { + ok = true + } } if ok { @@ -112,9 +118,15 @@ func (a *activeTask) processTask() { switch v := args.Data.(type) { case *requests.DNSRequest: go a.crawlName(args.Ctx, v, args.Params) + case *requests.AddrRequest: + if v.InScope && !a.enum.Config.NoCerts { + go a.certEnumeration(args.Ctx, v, args.Params) + } case *requests.ZoneXFRRequest: - go a.zoneTransfer(args.Ctx, v, args.Params) - go a.zoneWalk(args.Ctx, v, args.Params) + if !a.enum.Config.NoAxfr { + go a.zoneTransfer(args.Ctx, v, args.Params) + go a.zoneWalk(args.Ctx, v, args.Params) + } } } } @@ -164,6 +176,33 @@ func (a *activeTask) crawlName(ctx context.Context, req *requests.DNSRequest, tp } } +func (a *activeTask) certEnumeration(ctx context.Context, req *requests.AddrRequest, tp pipeline.TaskParams) { + defer func() { a.tokenPool <- struct{}{} }() + + if req == nil || !req.Valid() { + return + } + + for _, name := range http.PullCertificateNames(ctx, req.Address, a.enum.Config.Ports) { + select { + case <-ctx.Done(): + return + default: + } + + if n := strings.TrimSpace(name); n != "" { + if domain := a.enum.Config.WhichDomain(n); domain != "" { + pipeline.SendData(ctx, "new", &requests.DNSRequest{ + Name: n, + Domain: domain, + Tag: requests.CERT, + Source: "Active Cert", + }, tp) + } + } + } +} + func (a *activeTask) zoneTransfer(ctx context.Context, req *requests.ZoneXFRRequest, tp pipeline.TaskParams) { select { case <-ctx.Done(): diff --git a/enum/certs.go b/enum/certs.go deleted file mode 100644 index e26528a9b..000000000 --- a/enum/certs.go +++ /dev/null @@ -1,134 +0,0 @@ -package enum - -import ( - "context" - "github.com/OWASP/Amass/v3/net/http" - "github.com/OWASP/Amass/v3/requests" - "github.com/caffix/pipeline" - "github.com/caffix/queue" - "strings" -) - -// certsTask is the task that handles all requests related to ssl extraction within the pipeline. -type certsTask struct { - enum *Enumeration - queue queue.Queue - tokenPool chan struct{} -} - -type certsTaskArgs struct { - Ctx context.Context - Data pipeline.Data - Params pipeline.TaskParams -} - -// newCertsTask returns a certsTask specific to the provided Enumeration. -func newCertsTask(e *Enumeration, max int) *certsTask { - if max <= 0 { - return nil - } - - tokenPool := make(chan struct{}, max) - for i := 0; i < max; i++ { - tokenPool <- struct{}{} - } - - a := &certsTask{ - enum: e, - queue: queue.NewQueue(), - tokenPool: tokenPool, - } - - go a.ProcessQueue() - return a -} - -func (c *certsTask) Stop() { - c.queue.Process(func(e interface{}) {}) -} - -func (c *certsTask) ProcessQueue() { - for { - select { - case <-c.enum.done: - return - case <-c.queue.Signal(): - c.processTask() - } - } -} - -// Process implements the pipeline Task interface. -func (c *certsTask) Process(ctx context.Context, data pipeline.Data, tp pipeline.TaskParams) (pipeline.Data, error) { - select { - case <-ctx.Done(): - return nil, nil - default: - } - - var ok bool - switch data.(type) { - case *requests.AddrRequest: - ok = true - } - - if ok { - c.queue.Append(&certsTaskArgs{ - Ctx: ctx, - Data: data.Clone(), - Params: tp, - }) - } - - return data, nil -} - -func (c *certsTask) processTask() { - select { - case <-c.enum.ctx.Done(): - return - case <-c.enum.done: - return - case <-c.tokenPool: - element, ok := c.queue.Next() - if !ok { - c.tokenPool <- struct{}{} - return - } - - args := element.(*certsTaskArgs) - switch v := args.Data.(type) { - case *requests.AddrRequest: - if v.InScope { - go c.certEnumeration(args.Ctx, v, args.Params) - } - } - } -} - -func (c *certsTask) certEnumeration(ctx context.Context, req *requests.AddrRequest, tp pipeline.TaskParams) { - defer func() { c.tokenPool <- struct{}{} }() - - if req == nil || !req.Valid() { - return - } - - for _, name := range http.PullCertificateNames(ctx, req.Address, c.enum.Config.Ports) { - select { - case <-ctx.Done(): - return - default: - } - - if n := strings.TrimSpace(name); n != "" { - if domain := c.enum.Config.WhichDomain(n); domain != "" { - pipeline.SendData(ctx, "new", &requests.DNSRequest{ - Name: n, - Domain: domain, - Tag: requests.CERT, - Source: "Active Cert", - }, tp) - } - } - } -} diff --git a/enum/enum.go b/enum/enum.go index fe266eb83..f5e228336 100644 --- a/enum/enum.go +++ b/enum/enum.go @@ -120,12 +120,6 @@ func (e *Enumeration) Start(ctx context.Context) error { stages = append(stages, pipeline.FIFO("active", activetask)) } - if e.Config.Certs { - certstask := newCertsTask(e, maxActivePipelineTasks) - defer certstask.Stop() - - stages = append(stages, pipeline.FIFO("certs", certstask)) - } /* * Now that the pipeline input source has been setup, names provided From 7a30ab247f04b94ca1b19f0bf5750fb7585889af Mon Sep 17 00:00:00 2001 From: jep Date: Tue, 27 Jul 2021 18:34:48 +0200 Subject: [PATCH 3/3] guide examples --- doc/user_guide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/user_guide.md b/doc/user_guide.md index b2af46ebb..256d89325 100644 --- a/doc/user_guide.md +++ b/doc/user_guide.md @@ -131,8 +131,8 @@ This subcommand will perform DNS enumeration and network mapping while populatin | -src | Print data sources for the discovered names | amass enum -src -d example.com | | -timeout | Number of minutes to execute the enumeration | amass enum -timeout 30 -d example.com | | -w | Path to a different wordlist file | amass enum -brute -w wordlist.txt -d example.com | -| -nocerts | Disables certificate name grabs when -active mode is enabled | amass enum -active --no-certs -d example.com -| -noaxfr | Disables zone transfers when -active mode is enabled | amass enum -active -no-axfr -d example.com +| -nocerts | Disables certificate name grabs when -active mode is enabled | amass enum -active -nocerts -d example.com +| -noaxfr | Disables zone transfers when -active mode is enabled | amass enum -active -noaxfr -d example.com ### The 'viz' Subcommand