Merge pull request #692 from projectdiscovery/dev

v1.2.3 Release

Author: ehsandeep

.github/workflows/sonarcloud.yml

@@ -27,9 +27,9 @@ jobs:
           go test -coverprofile=./cov.out ./...
 
       - name: Run Gosec Security Scanner
-        run: |
-          go install github.com/securego/gosec/cmd/gosec@latest
-          gosec -no-fail -fmt=sonarqube -out report.json ./...
+        uses: securego/gosec@master
+        with:
+          args: '-no-fail -fmt=sonarqube -out report.json ./...'
 
       - name: SonarCloud Scan
         uses: SonarSource/sonarcloud-github-action@master

README.md

@@ -116,11 +116,12 @@ MATCHERS:
    -mfc, -match-favicon string[]      match response with specified favicon hash (-mfc 1494302000)
    -ms, -match-string string          match response with specified string (-ms admin)
    -mr, -match-regex string           match response with specified regex (-mr admin)
-   -mcdn, -match-cdn string[]         match host with specified cdn provider (google, azure, cloudflare, cloudfront, fastly, incapsula, oracle, akamai, sucuri, leaseweb)
+   -mcdn, -match-cdn string[]         match host with specified cdn provider (azure, cloudflare, cloudfront, fastly, incapsula, oracle, google, akamai, sucuri, leaseweb)
    -mrt, -match-response-time string  match response with specified response time in seconds (-mrt '< 1')
 
 EXTRACTOR:
-   -er, -extract-regex string  display response content for specified regex
+   -er, -extract-regex string[]   Display response content with matched regex
+   -ep, -extract-preset string[]  Display response content with matched preset regex
 
 FILTERS:
    -fc, -filter-code string            filter response with specified status code (-fc 403,401)
@@ -130,7 +131,7 @@ FILTERS:
    -ffc, -filter-favicon string[]      filter response with specified favicon hash (-mfc 1494302000)
    -fs, -filter-string string          filter response with specified string (-fs admin)
    -fe, -filter-regex string           filter response with specified regex (-fe admin)
-   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (google, azure, cloudflare, cloudfront, fastly, incapsula, oracle, akamai, sucuri, leaseweb)
+   -fcdn, -filter-cdn string[]         filter host with specified cdn provider (azure, cloudflare, cloudfront, fastly, incapsula, oracle, google, akamai, sucuri, leaseweb)
    -frt, -filter-response-time string  filter response with specified response time in seconds (-frt '> 1')
 
 RATE-LIMIT:
@@ -180,6 +181,7 @@ CONFIGURATIONS:
    -ldp, -leave-default-ports    leave default http/https ports in host header (eg. http://host:80 - https//host:443
 
 DEBUG:
+   -health-check, -hc        run diagnostic check up
    -debug                    display request/response content in cli
    -debug-req                display request content in cli
    -debug-resp               display response content in cli
@@ -395,7 +397,7 @@ https://api.hackerone.com [AS13335, CLOUDFLARENET, US, 104.16.96.0/20]
 ```
 
 
-### Path Probe
+### File/Path Bruteforce
 
 
 ```console
@@ -451,8 +453,47 @@ https://docs.hackerone.com
 https://support.hackerone.com
 ```
 
+### Using httpx as a library
+`httpx` can be used as a library by creating an instance of the `Option` struct and populating it with the same options that would be specified via CLI. Once validated, the struct should be passed to a runner instance (to close at the end of the program) and the `RunEnumeration` method should be called. Here follows a minimal example of how to do it:
+
+```go
+package main
+
+import (
+	"log"
+	"os"
+
+	"github.com/projectdiscovery/httpx/runner"
+)
+
+func main() {
+	inputFile := "test.txt"
+	err := os.WriteFile(inputFile, []byte("scanme.sh"), 0644)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer os.RemoveAll(inputFile)
+
+	options := runner.Options{
+		Methods:   "GET",
+		InputFile: inputFile,
+	}
+	if err := options.ValidateOptions(); err != nil {
+		log.Fatal(err)
+	}
+
+	httpxRunner, err := runner.New(&options)
+	if err != nil {
+		log.Fatal()
+	}
+	defer httpxRunner.Close()
+
+	httpxRunner.RunEnumeration()
+}
+```
+
 
-# 📋 Notes
+# Notes
 
 - As default, **httpx** checks for `HTTPS` probe and fall-back to `HTTP` only if `HTTPS` is not reachable.
 - For printing both HTTP/HTTPS results, `no-fallback` flag can be used.

cmd/functional-test/run.sh

@@ -11,13 +11,13 @@ echo "::group::Building functional-test binary"
 go build -o functional-test$extension
 echo "::endgroup::"
 
-echo "::group::Building dnsx binary from current branch"
+echo "::group::Building httpx binary from current branch"
 go build -o httpx_dev$extension ../httpx
 echo "::endgroup::"
 
-echo "::group::Building latest release of dnsx"
+echo "::group::Building latest release of httpx"
 go build -o httpx$extension -v github.com/projectdiscovery/httpx/cmd/httpx
 echo "::endgroup::"
 
-echo 'Starting dnsx functional test'
+echo 'Starting httpx functional test'
 ./functional-test$extension -main ./httpx$extension -dev ./httpx_dev$extension -testcases testcases.txt

cmd/functional-test/test-data/request.txt

@@ -1 +1 @@
-https://www.example.com
+https://scanme.sh

cmd/functional-test/testcases.txt

@@ -1,16 +1,16 @@
-www.example.com {{binary}} -silent 
-www.example.com {{binary}} -silent -l test-data/request.txt
-www.example.com {{binary}} -silent -request test-data/raw-request.txt
-www.example.com {{binary}} -silent -title
-www.example.com {{binary}} -silent -sc
-www.example.com {{binary}} -silent -td
-www.example.com {{binary}} -silent -probe
-www.example.com {{binary}} -silent -no-fallback
-www.example.com {{binary}} -silent -cl
-www.example.com {{binary}} -silent -server
-www.example.com {{binary}} -silent -ip
-www.example.com {{binary}} -silent -tls-grab
-www.example.com {{binary}} -silent -unsafe
-www.example.com {{binary}} -silent -x all
-www.example.com {{binary}} -silent -body 'a=b'
-www.example.com {{binary}} -silent -exclude-cdn
+scanme.sh {{binary}} -silent 
+scanme.sh {{binary}} -silent -l test-data/request.txt
+scanme.sh {{binary}} -silent -request test-data/raw-request.txt
+scanme.sh {{binary}} -silent -title
+scanme.sh {{binary}} -silent -sc
+scanme.sh {{binary}} -silent -td
+scanme.sh {{binary}} -silent -probe
+scanme.sh {{binary}} -silent -no-fallback
+scanme.sh {{binary}} -silent -cl
+scanme.sh {{binary}} -silent -server
+scanme.sh {{binary}} -silent -ip
+scanme.sh {{binary}} -silent -tls-grab
+scanme.sh {{binary}} -silent -unsafe
+scanme.sh {{binary}} -silent -x all
+scanme.sh {{binary}} -silent -body 'a=b'
+scanme.sh {{binary}} -silent -exclude-cdn

cmd/integration-test/integration-test.go

@@ -22,7 +22,8 @@ func main() {
 	failed := aurora.Red("[✘]").String()
 
 	tests := map[string]map[string]testutils.TestCase{
-		"http": httpTestcases,
+		"http":    httpTestcases,
+		"library": libraryTestcases,
 	}
 	for proto, tests := range tests {
 		if protocol == "" || protocol == proto {

cmd/integration-test/library.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"os"
+
+	"github.com/projectdiscovery/httpx/internal/testutils"
+	"github.com/projectdiscovery/httpx/runner"
+)
+
+var libraryTestcases = map[string]testutils.TestCase{
+	"Httpx as library": &httpxLibrary{},
+}
+
+type httpxLibrary struct {
+}
+
+func (h *httpxLibrary) Execute() error {
+	testFile := "test.txt"
+	err := os.WriteFile(testFile, []byte("scanme.sh"), 0644)
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(testFile)
+
+	options := runner.Options{
+		Methods:   "GET",
+		InputFile: testFile,
+	}
+	if err := options.ValidateOptions(); err != nil {
+		return err
+	}
+
+	httpxRunner, err := runner.New(&options)
+	if err != nil {
+		return err
+	}
+	defer httpxRunner.Close()
+
+	httpxRunner.RunEnumeration()
+	return nil
+}

common/customextract/customextract.go

@@ -0,0 +1,9 @@
+package customextract
+
+import "regexp"
+
+var ExtractPresets = map[string]*regexp.Regexp{
+	"url":  regexp.MustCompile("^(http(s)?:\\/\\/)[a-zA-Z0-9][-a-zA-Z0-9]{0,62}(\\.[a-zA-Z0-9][-a-zA-Z0-9]{0,62})+(:[0-9]{1,5})?[-a-zA-Z0-9()@:%_\\\\\\+\\.~#?&//=]*$"), //nolint
+	"ip":   regexp.MustCompile("((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})(\\.((2(5[0-5]|[0-4]\\d))|[0-1]?\\d{1,2})){3}"),                                                    //nolint
+	"mail": regexp.MustCompile("^([A-Za-z0-9_\\-\\.\u4e00-\u9fa5])+\\@([A-Za-z0-9_\\-\\.])+\\.([A-Za-z]{2,8})$"),                                                        //nolint
+}

common/customextract/doc.go

@@ -0,0 +1 @@
+package customextract

common/customports/customport.go

@@ -1,6 +1,7 @@
 package customport
 
 import (
+	"fmt"
 	"strconv"
 	"strings"
 
@@ -28,6 +29,36 @@ func (c *CustomPorts) String() string {
 	return "Custom Ports"
 }
 
+// ValidateCustomPorts to validate the custom port range
+func ValidateCustomPorts(value string) error {
+	potentialRange := strings.Split(value, "-")
+	if len(potentialRange) < portRangeParts {
+		if _, err := strconv.Atoi(value); err != nil {
+			return fmt.Errorf("Could not cast port to integer from your value: %s. Resulting error: %s.\n", value, err.Error())
+		}
+	} else {
+		part1 := potentialRange[0]
+		part2 := potentialRange[1]
+		lowP, err := strconv.Atoi(part1)
+		if err != nil {
+			return fmt.Errorf("Could not cast first port of your range(%s) to integer from your value: %s. Resulting error: %s.\n",
+				value, part1, err.Error())
+		}
+		highP, err := strconv.Atoi(part2)
+		if err != nil {
+			return fmt.Errorf("Could not cast last port of your port range(%s) to integer from "+
+				"your value: %s. Resulting error %s.\n",
+				value, part2, err.Error())
+		}
+		if lowP > highP {
+			return fmt.Errorf("First value of port range should be lower than the last port "+
+				"from your range: [%d, %d].\n",
+				lowP, highP)
+		}
+	}
+	return nil
+}
+
 // Set a port range
 func (c *CustomPorts) Set(value string) error {
 	// ports can be like nmap -p [https|http:]start-end,[https|http:]port1,[https|http:]port2,[https|http:]port3