WARNING: THIS SITE IS A MIRROR OF GITHUB.COM / IT CANNOT LOGIN OR REGISTER ACCOUNTS / THE CONTENTS ARE PROVIDED AS-IS / THIS SITE ASSUMES NO RESPONSIBILITY FOR ANY DISPLAYED CONTENT OR LINKS / IF YOU FOUND SOMETHING MAY NOT GOOD FOR EVERYONE, CONTACT ADMIN AT ilovescratch@foxmail.com
Skip to content

Commit cc04a59

Browse files
Merge pull request #1668 from projectdiscovery/dev
v2.10.0
2 parents f5fe4f8 + 5bcfca5 commit cc04a59

File tree

16 files changed

+672
-184
lines changed

16 files changed

+672
-184
lines changed

.github/workflows/build-test.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@ jobs:
5858
FOFA_API_KEY: ${{secrets.FOFA_API_KEY}}
5959
FULLHUNT_API_KEY: ${{secrets.FULLHUNT_API_KEY}}
6060
GITHUB_API_KEY: ${{secrets.GITHUB_API_KEY}}
61-
HUNTER_API_KEY: ${{secrets.HUNTER_API_KEY}}
6261
INTELX_API_KEY: ${{secrets.INTELX_API_KEY}}
6362
LEAKIX_API_KEY: ${{secrets.LEAKIX_API_KEY}}
6463
QUAKE_API_KEY: ${{secrets.QUAKE_API_KEY}}

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,5 @@ vendor/
88
.idea
99
.devcontainer
1010
.vscode
11-
dist
11+
dist
12+
/subfinder

.goreleaser.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ builds:
1313
- darwin
1414
goarch:
1515
- amd64
16-
- 386
16+
- '386'
1717
- arm
1818
- arm64
1919

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ require (
7373
github.com/minio/selfupdate v0.6.1-0.20230907112617-f11e74f84ca7 // indirect
7474
github.com/muesli/reflow v0.3.0 // indirect
7575
github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a // indirect
76-
github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect
76+
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
7777
github.com/pierrec/lz4/v4 v4.1.21 // indirect
7878
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
7979
github.com/projectdiscovery/blackrock v0.0.1 // indirect

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -231,8 +231,8 @@ github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
231231
github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
232232
github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a h1:2MaM6YC3mGu54x+RKAA6JiFFHlHDY1UbkxqppT7wYOg=
233233
github.com/muesli/termenv v0.15.3-0.20240618155329-98d742f6907a/go.mod h1:hxSnBBYLK21Vtq/PHd0S2FYCxBXzBua8ov5s1RobyRQ=
234-
github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY=
235-
github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=
234+
github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A=
235+
github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
236236
github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
237237
github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
238238
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

pkg/passive/sources.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus"
2626
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb"
2727
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster"
28+
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/domainsproject"
2829
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo"
2930
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/driftnet"
3031
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/facebook"
@@ -33,10 +34,10 @@ import (
3334
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github"
3435
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget"
3536
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hudsonrock"
36-
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hunter"
3737
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx"
3838
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix"
3939
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas"
40+
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onyphe"
4041
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon"
4142
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake"
4243
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns"
@@ -51,6 +52,7 @@ import (
5152
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/virustotal"
5253
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive"
5354
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi"
55+
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/windvane"
5456
"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi"
5557
mapsutil "github.com/projectdiscovery/utils/maps"
5658
)
@@ -70,15 +72,16 @@ var AllSources = [...]subscraping.Source{
7072
&digitorus.Source{},
7173
&dnsdb.Source{},
7274
&dnsdumpster.Source{},
75+
&domainsproject.Source{},
7376
&dnsrepo.Source{},
7477
&driftnet.Source{},
7578
&fofa.Source{},
7679
&fullhunt.Source{},
7780
&github.Source{},
7881
&hackertarget.Source{},
79-
&hunter.Source{},
8082
&intelx.Source{},
8183
&netlas.Source{},
84+
&onyphe.Source{},
8285
&leakix.Source{},
8386
&quake.Source{},
8487
&pugrecon.Source{},
@@ -95,6 +98,7 @@ var AllSources = [...]subscraping.Source{
9598
&virustotal.Source{},
9699
&waybackarchive.Source{},
97100
&whoisxmlapi.Source{},
101+
&windvane.Source{},
98102
&zoomeyeapi.Source{},
99103
&facebook.Source{},
100104
// &threatminer.Source{}, // failing api

pkg/passive/sources_test.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,15 @@ var (
2626
"dnsdumpster",
2727
"dnsdb",
2828
"dnsrepo",
29+
"domainsproject",
2930
"driftnet",
3031
"fofa",
3132
"fullhunt",
3233
"github",
3334
"hackertarget",
3435
"intelx",
3536
"netlas",
37+
"onyphe",
3638
"quake",
3739
"pugrecon",
3840
"rapiddns",
@@ -48,8 +50,8 @@ var (
4850
"virustotal",
4951
"waybackarchive",
5052
"whoisxmlapi",
53+
"windvane",
5154
"zoomeyeapi",
52-
"hunter",
5355
"leakix",
5456
"facebook",
5557
// "threatminer",
@@ -72,22 +74,24 @@ var (
7274
"crtsh",
7375
"digitorus",
7476
"dnsdumpster",
77+
"domainsproject",
7578
"dnsrepo",
7679
"driftnet",
7780
"fofa",
7881
"fullhunt",
7982
"hackertarget",
8083
"intelx",
84+
"onyphe",
8185
"quake",
8286
"redhuntlabs",
8387
"robtex",
8488
// "riddler", // failing due to cloudfront protection
8589
"rsecloud",
8690
"securitytrails",
8791
"shodan",
92+
"windvane",
8893
"virustotal",
8994
"whoisxmlapi",
90-
"hunter",
9195
"leakix",
9296
"facebook",
9397
// "threatminer",

pkg/resolve/resolve.go

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,20 @@ type ResolutionPool struct {
2525

2626
// HostEntry defines a host with the source
2727
type HostEntry struct {
28-
Domain string
29-
Host string
30-
Source string
28+
Domain string
29+
Host string
30+
Source string
31+
WildcardCertificate bool
3132
}
3233

3334
// Result contains the result for a host resolution
3435
type Result struct {
35-
Type ResultType
36-
Host string
37-
IP string
38-
Error error
39-
Source string
36+
Type ResultType
37+
Host string
38+
IP string
39+
Error error
40+
Source string
41+
WildcardCertificate bool
4042
}
4143

4244
// ResultType is the type of result found
@@ -92,13 +94,13 @@ func (r *ResolutionPool) InitWildcards(domain string) error {
9294
func (r *ResolutionPool) resolveWorker() {
9395
for task := range r.Tasks {
9496
if !r.removeWildcard {
95-
r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source}
97+
r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source, WildcardCertificate: task.WildcardCertificate}
9698
continue
9799
}
98100

99101
hosts, err := r.DNSClient.Lookup(task.Host)
100102
if err != nil {
101-
r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err}
103+
r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err, WildcardCertificate: task.WildcardCertificate}
102104
continue
103105
}
104106

@@ -116,7 +118,7 @@ func (r *ResolutionPool) resolveWorker() {
116118
}
117119

118120
if !skip {
119-
r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source}
121+
r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source, WildcardCertificate: task.WildcardCertificate}
120122
}
121123
}
122124
r.wg.Done()

pkg/runner/banners.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ const banner = `
1717
const ToolName = `subfinder`
1818

1919
// Version is the current version of subfinder
20-
const version = `v2.9.0`
20+
const version = `v2.9.1-dev`
2121

2222
// showBanner is used to show the banner to the user
2323
func showBanner() {

pkg/runner/enumerate.go

Lines changed: 28 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
6767
gologger.Warning().Msgf("Encountered an error with source %s: %s\n", result.Source, result.Error)
6868
case subscraping.Subdomain:
6969
subdomain := replacer.Replace(result.Value)
70+
// check if this subdomain is actually a wildcard subdomain
71+
// that may have furthur subdomains associated with it
72+
isWildcard := strings.Contains(result.Value, "*."+subdomain)
7073

7174
// Validate the subdomain found and remove wildcards from
7275
if !strings.HasSuffix(subdomain, "."+domain) {
@@ -90,10 +93,20 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
9093
// send the subdomain for resolution.
9194
if _, ok := uniqueMap[subdomain]; ok {
9295
skippedCounts[result.Source]++
96+
// even if it is duplicate if it was not marked as wildcard before but this source says it is wildcard
97+
// then we should mark it as wildcard
98+
if !uniqueMap[subdomain].WildcardCertificate && isWildcard {
99+
val := uniqueMap[subdomain]
100+
val.WildcardCertificate = true
101+
uniqueMap[subdomain] = val
102+
}
93103
continue
94104
}
95105

96-
hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source}
106+
hostEntry := resolve.HostEntry{Domain: domain, Host: subdomain, Source: result.Source, WildcardCertificate: isWildcard}
107+
if r.options.ResultCallback != nil && !r.options.RemoveWildcard {
108+
r.options.ResultCallback(&hostEntry)
109+
}
97110

98111
uniqueMap[subdomain] = hostEntry
99112
// If the user asked to remove wildcard then send on the resolve
@@ -109,6 +122,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
109122
if r.options.RemoveWildcard {
110123
close(resolutionPool.Tasks)
111124
}
125+
112126
wg.Done()
113127
}()
114128

@@ -125,9 +139,22 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
125139
// Add the found subdomain to a map.
126140
if _, ok := foundResults[result.Host]; !ok {
127141
foundResults[result.Host] = result
142+
if r.options.ResultCallback != nil {
143+
r.options.ResultCallback(&resolve.HostEntry{Domain: domain, Host: result.Host, Source: result.Source, WildcardCertificate: result.WildcardCertificate})
144+
}
128145
}
129146
}
130147
}
148+
149+
// Merge wildcard certificate information from uniqueMap into foundResults
150+
// This handles cases where a later source marked a subdomain as wildcard
151+
// after it was already sent to the resolution pool
152+
for host, result := range foundResults {
153+
if entry, ok := uniqueMap[host]; ok && entry.WildcardCertificate && !result.WildcardCertificate {
154+
result.WildcardCertificate = true
155+
foundResults[host] = result
156+
}
157+
}
131158
}
132159
wg.Wait()
133160
outputWriter := NewOutputWriter(r.options.JSON)
@@ -162,17 +189,6 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
162189
numberOfSubDomains = len(uniqueMap)
163190
}
164191

165-
if r.options.ResultCallback != nil {
166-
if r.options.RemoveWildcard {
167-
for host, result := range foundResults {
168-
r.options.ResultCallback(&resolve.HostEntry{Domain: host, Host: result.Host, Source: result.Source})
169-
}
170-
} else {
171-
for _, v := range uniqueMap {
172-
r.options.ResultCallback(&v)
173-
}
174-
}
175-
}
176192
gologger.Info().Msgf("Found %d subdomains for %s in %s\n", numberOfSubDomains, domain, duration)
177193

178194
if r.options.Statistics {

0 commit comments

Comments
 (0)