From d8a2d9c4b25bceb1e0f6b18dbb033f0bfa49455c Mon Sep 17 00:00:00 2001 From: David Dick Date: Sun, 5 May 2024 18:16:01 +1000 Subject: [PATCH] Using mdn/browser-compat-data for stealth --- .gitmodules | 3 + MANIFEST | 2 + README | 24 +- README.md | 13 +- browserfeatcl | 1 + build-bcd-for-firefox | 449 +++++++++++++ lib/Firefox/Marionette.pm | 89 ++- lib/Firefox/Marionette/Extension/Stealth.pm | 694 +++++++++++++++++--- t/01-marionette.t | 255 ------- t/04-browserfeatcl.t | 497 ++++++++++++++ t/author/bulk_test.pl | 1 + t/manifest.t | 1 + t/stub.pl | 20 +- t/syscall_tests.pm | 7 +- t/test_daemons.pm | 58 +- 15 files changed, 1707 insertions(+), 407 deletions(-) create mode 160000 browserfeatcl create mode 100755 build-bcd-for-firefox create mode 100644 t/04-browserfeatcl.t diff --git a/.gitmodules b/.gitmodules index b9335fa..dc5d1da 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "fingerprintjs"] path = fingerprintjs url = https://github.com/fingerprintjs/fingerprintjs.git +[submodule "browserfeatcl"] + path = browserfeatcl + url = https://github.com/lraj22/browserfeatcl.git diff --git a/MANIFEST b/MANIFEST index 66090dd..5500cea 100644 --- a/MANIFEST +++ b/MANIFEST @@ -41,6 +41,7 @@ MANIFEST This list of files LICENSE README README.md +build-bcd-for-firefox ca-bundle-for-firefox check-firefox-certificate-authorities mozilla-head-check @@ -80,6 +81,7 @@ t/03-seek.t t/03-stat.t t/03-sysopen.t t/04-botd.t +t/04-browserfeatcl.t t/04-fingerprint.t t/04-proxy.t t/04-webauthn.t diff --git a/README b/README index c0f5dd8..f049378 100644 --- a/README +++ b/README @@ -34,6 +34,15 @@ DESCRIPTION Marionette protocol +CONSTANTS + + BCD_PATH + + returns the local path used for storing the brower compability data for + the agent method when the stealth parameter is supplied to + the new method. This database is built by the build-bcd-for-firefox + binary. + SUBROUTINES/METHODS accept_alert @@ -445,9 +454,10 @@ SUBROUTINES/METHODS # Mozilla/5.0 (X11; Linux s390x; rv:109.0) Gecko/20100101 Firefox/115.0 If the stealth parameter has supplied to the new method, it will also - attempt to change a number of javascript attributes to match the - desired browser. The following websites have been very useful in - testing these ideas; + attempt to delete/provide dummy implementations for number of + javascript attributes to + match the desired browser. The following websites have been very useful + in testing these ideas; * https://browserleaks.com/javascript @@ -455,6 +465,12 @@ SUBROUTINES/METHODS * https://bot.sannysoft.com/ + * https://lraj22.github.io/browserfeatcl/ + + Importantly, this will break feature detection + + for any website that relies on it. + See IMITATING OTHER BROWSERS a discussion of these types of techniques. These changes are not foolproof, but it is interesting to see what can be done with modern browsers. All this behaviour should be regarded as @@ -3512,7 +3528,7 @@ WEBSITES THAT BLOCK AUTOMATION If the web site you are trying to automate mysteriously fails when you are automating a workflow, but it works when you perform the workflow manually, you may be dealing with a web site that is hostile to - automation. + automation. I would be very interested if you can supply a test case. At the very least, under these circumstances, it would be a good idea to be aware that there's an ongoing arms race diff --git a/README.md b/README.md index e3ebadf..0c67ee9 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,12 @@ Version 1.55 This is a client module to automate the Mozilla Firefox browser via the [Marionette protocol](https://developer.mozilla.org/en-US/docs/Mozilla/QA/Marionette/Protocol) +# CONSTANTS + +## BCD\_PATH + +returns the local path used for storing the brower compability data for the [agent](#agent) method when the <code>stealth</code> parameter is supplied to the [new](#new) method. This database is built by the build-bcd-for-firefox binary. + # SUBROUTINES/METHODS ## accept\_alert @@ -294,11 +300,14 @@ These parameters can be used to set a user agent string like so; # user agent is now equal to # Mozilla/5.0 (X11; Linux s390x; rv:109.0) Gecko/20100101 Firefox/115.0 -If the `stealth` parameter has supplied to the [new](#new) method, it will also attempt to change a number of javascript attributes to match the desired browser. The following websites have been very useful in testing these ideas; +If the `stealth` parameter has supplied to the [new](#new) method, it will also attempt to delete/provide dummy implementations for number of [javascript attributes](https://github.com/mdn/browser-compat-data) to match the desired browser. The following websites have been very useful in testing these ideas; - [https://browserleaks.com/javascript](https://browserleaks.com/javascript) - [https://www.amiunique.org/fingerprint](https://www.amiunique.org/fingerprint) - [https://bot.sannysoft.com/](https://bot.sannysoft.com/) +- [https://lraj22.github.io/browserfeatcl/](https://lraj22.github.io/browserfeatcl/) + +Importantly, this will break [feature detection](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Feature_detection) for any website that relies on it. See [IMITATING OTHER BROWSERS](#imitating-other-browsers) a discussion of these types of techniques. These changes are not foolproof, but it is interesting to see what can be done with modern browsers. All this behaviour should be regarded as extremely experimental and subject to change. Feedback welcome. @@ -2493,7 +2502,7 @@ This list of methods may grow. Marionette [by design](https://developer.mozilla.org/en-US/docs/Web/API/Navigator/webdriver) allows web sites to detect that the browser is being automated. Firefox [no longer (since version 88)](https://bugzilla.mozilla.org/show_bug.cgi?id=1632821) allows you to disable this functionality while you are automating the browser, but this can be overridden with the `stealth` parameter for the [new](#new) method. This is extremely experimental and feedback is welcome. -If the web site you are trying to automate mysteriously fails when you are automating a workflow, but it works when you perform the workflow manually, you may be dealing with a web site that is hostile to automation. +If the web site you are trying to automate mysteriously fails when you are automating a workflow, but it works when you perform the workflow manually, you may be dealing with a web site that is hostile to automation. I would be very interested if you can supply a test case. At the very least, under these circumstances, it would be a good idea to be aware that there's an [ongoing arms race](https://en.wikipedia.org/wiki/Web_scraping#Methods_to_prevent_web_scraping), and potential [legal issues](https://en.wikipedia.org/wiki/Web_scraping#Legal_issues) in this area. diff --git a/browserfeatcl b/browserfeatcl new file mode 160000 index 0000000..2e11f83 --- /dev/null +++ b/browserfeatcl @@ -0,0 +1 @@ +Subproject commit 2e11f83f725c7f5303065d3b9a158554e2a401b2 diff --git a/build-bcd-for-firefox b/build-bcd-for-firefox new file mode 100755 index 0000000..9354f6e --- /dev/null +++ b/build-bcd-for-firefox @@ -0,0 +1,449 @@ +#! /usr/bin/perl + +use strict; +use warnings; +use Getopt::Long qw(:config bundling); +use Cwd(); +use File::Spec(); +use File::Find(); +use JSON(); +use English qw( -no_match_vars ); +use Fcntl(); +use FileHandle(); +use File::HomeDir(); +use File::Temp(); +use JSON(); +use Carp(); +use Firefox::Marionette(); + +local $ENV{PATH} = '/usr/bin:/bin:/usr/sbin:/sbin'; +delete @ENV{qw(IFS CDPATH ENV BASH_ENV)}; + +sub _BUFFER_SIZE { return 8192 } +sub _MAX_KEYS { return 3 } + +our $VERSION = '1.55'; + +MAIN: { + my %options; + + Getopt::Long::GetOptions( \%options, 'help|h', 'version|v', 'path|p', ); + my $bcd_path = Firefox::Marionette::BCD_PATH(1); + _parse_options( $bcd_path, %options ); + my ( $volume, $directories, $file ) = File::Spec->splitpath($bcd_path); + my $firefox_marionette_directory = + File::Spec->catdir( $volume, $directories ); + my $browser_compat_data_directory = + File::Spec->catdir( $firefox_marionette_directory, + 'browser-compat-data' ); + _setup_git_repos($browser_compat_data_directory); + my $summary = {}; + my $api_directory = + File::Spec->catdir( $browser_compat_data_directory, 'api' ); + my $builtins_directory = File::Spec->catdir( $browser_compat_data_directory, + 'javascript', 'builtins' ); + my $debug = 0; + my $firefox = Firefox::Marionette->new( debug => $debug )->content() + ->go('https://duckduckgo.com'); + File::Find::find( + { + wanted => sub { + if ( $File::Find::name =~ /[.]json$/smx ) { + if ($debug) { Carp::carp("Looking at $File::Find::name\n") } + my $path = $File::Find::name; + my $handle = FileHandle->new( $path, Fcntl::O_RDONLY() ) + or Carp::croak( + "Failed to open $path for reading:$EXTENDED_OS_ERROR"); + my $content; + my $result; + while ( $result = sysread $handle, + my $buffer, _BUFFER_SIZE() ) + { + $content .= $buffer; + } + defined $result + or Carp::croak( + "Failed to read from $path:$EXTENDED_OS_ERROR"); + close $handle + or + Carp::croak("Failed to close $path:$EXTENDED_OS_ERROR"); + my $json = JSON::decode_json($content); + my $root_element = _get_root_element( $json, $path ); + foreach my $class_name ( + sort { $a cmp $b } + keys %{$root_element} + ) + { + foreach + my $browser (qw(firefox chrome edge safari ie opera)) + { + my $class_reference = + $root_element->{$class_name}->{__compat} + ->{support}->{$browser}; + my $mirror_reference = + $root_element->{$class_name}->{__compat} + ->{support}->{chrome}; + my @versions = _get_versions( $class_reference, + $mirror_reference, $class_name ); + foreach my $version ( + sort { + $a->{version_added} <=> $b->{version_added} + } @versions + ) + { + next if ( $version->{partial_implementation} ); + _process_version( + $summary, $class_name, $browser, + $version, $path + ); + } + foreach my $function_name ( + sort { $a cmp $b } + keys %{ $root_element->{$class_name} } + ) + { + next if ( $function_name eq '__compat' ); + next if ( $function_name eq 'worker_support' ); + _process_function( + $summary, + root_element => $root_element, + class_name => $class_name, + function_name => $function_name, + browser => $browser, + path => $path + ); + } + } + } + } + }, + follow => 1 + }, + $api_directory, + $builtins_directory + ); + my ( $bcd_handle, $tmp_path ) = File::Temp::tempfile( + 'firefox-marionette-bcd-XXXXXXXXXXX', + DIR => $firefox_marionette_directory + ); + $bcd_handle->print( JSON->new()->pretty()->encode($summary) ) + or Carp::croak("Failed to write to $tmp_path:$EXTENDED_OS_ERROR"); + $bcd_handle->close() + or Carp::croak("Failed to close $tmp_path:$EXTENDED_OS_ERROR"); + rename $tmp_path, $bcd_path + or + Carp::croak("Failed to rename $tmp_path to $bcd_path:$EXTENDED_OS_ERROR"); +} + +sub _parse_options { + my ( $bcd_path, %options ) = @_; + if ( $options{help} ) { + require Pod::Simple::Text; + my $parser = Pod::Simple::Text->new(); + $parser->parse_from_file($PROGRAM_NAME); + exit 0; + } + elsif ( $options{version} ) { + print "$VERSION\n" + or die "Failed to print to STDOUT:$EXTENDED_OS_ERROR\n"; + exit 0; + } + elsif ( $options{path} ) { + print "$bcd_path\n" + or die "Failed to print to STDOUT:$EXTENDED_OS_ERROR\n"; + exit 0; + } + return; +} + +sub _setup_git_repos { + my ($browser_compat_data_directory) = @_; + if ( -d $browser_compat_data_directory ) { + my $cwd = Cwd::cwd(); + chdir $browser_compat_data_directory + or Carp::croak( + "Failed to chdir $browser_compat_data_directory:$EXTENDED_OS_ERROR" + ); + system {'git'} 'git', 'pull', '--quiet' + and Carp::croak( + "Failed to git pull from $browser_compat_data_directory\n"); + chdir $cwd or Carp::croak("Failed to chdir $cwd:$EXTENDED_OS_ERROR"); + } + else { + my $mdn_browser_repo = 'https://github.com/mdn/browser-compat-data.git'; + system {'git'} 'git', 'clone', $mdn_browser_repo, + $browser_compat_data_directory + and Carp::croak( +"Failed to git clone $mdn_browser_repo $browser_compat_data_directory\n" + ); + } + return; +} + +sub _process_version { + my ( $summary, $class_name, $browser, $version, $path ) = @_; + if ( $version->{version_added} ) { + if ( $version->{version_added} ne 'preview' ) { + $summary->{$class_name}->{type} = 'class'; + my %extra = _get_extra_from_flags( $version, $path ); + push @{ $summary->{$class_name}->{browsers}->{$browser} }, + { add => $version->{version_added}, %extra }; + } + } + if ( $version->{version_removed} ) { + push @{ $summary->{$class_name}->{browsers}->{$browser} }, + { rm => $version->{version_removed} + 0 }; + } + return; +} + +sub _process_function { + my ( $summary, %parameters ) = @_; + my $root_element = $parameters{root_element}; + my $function_name = $parameters{function_name}; + my $class_name = $parameters{class_name}; + my $browser = $parameters{browser}; + my $path = $parameters{path}; + my $cleaned_function_name = $function_name; + my $static; + + if ( $cleaned_function_name =~ s/_static$//smx ) { + $static = 1; + } + my $function_reference = + $root_element->{$class_name}->{$function_name}->{__compat}->{support} + ->{$browser}; + my $mirror_reference = + $root_element->{$class_name}->{$function_name}->{__compat}->{support} + ->{chrome}; + my @versions = + _get_versions( $function_reference, $mirror_reference, $function_name ); + foreach my $version ( sort { $a->{version_added} <=> $b->{version_added} } + @versions ) + { +# Example of partial_implementation to allow property to exist HTMLAnchorElement.ping +# next if ( $version->{partial_implementation} ); +# Example of note to allow property to exist - AudioBufferSourceNode.buffer +# next if ( $version->{notes} ); + if ( ( $version->{version_added} ) + && ( $version->{version_added} ne 'preview' ) ) + { + $summary->{"$class_name.$cleaned_function_name"}->{type} = + 'function'; + $summary->{"$class_name.$cleaned_function_name"}->{static} = + $static ? \1 : \0; + my %extra = _get_extra_from_flags( $version, $path ); + push @{ $summary->{"$class_name.$cleaned_function_name"}->{browsers} + ->{$browser} }, { add => $version->{version_added}, %extra }; + } + if ( ( $version->{version_removed} ) + && ( $version->{version_removed} ne 'preview' ) ) + { + push @{ $summary->{"$class_name.$cleaned_function_name"}->{browsers} + ->{$browser} }, { rm => $version->{version_removed} + 0 }; + } + } + return; +} + +sub _get_extra_from_flags { + my ( $version, $path ) = @_; + my %extra; + if ( $version->{flags} ) { + foreach my $flag ( @{ $version->{flags} } ) { + if ( ( defined $flag->{type} ) + && ( defined $flag->{name} ) + && ( defined $flag->{value_to_set} ) + && ( ( keys %{$flag} ) == _MAX_KEYS() ) ) + { + } + elsif (( defined $flag->{type} ) + && ( defined $flag->{name} ) + && ( !defined $flag->{value_to_set} ) + && ( ( keys %{$flag} ) == 2 ) ) + { + next; + } + else { + Carp::croak("Unknown flag for $path"); + } + if ( $flag->{type} eq 'preference' ) { + %extra = ( + pref_name => $flag->{name}, + pref_value => $flag->{value_to_set} + ); + } + else { + Carp::croak("Unknown type of '$flag->{type}' in $path"); + } + } + } + return %extra; +} + +sub _get_root_element { + my ( $json, $path ) = @_; + my $root_element; + my @parts; + while ( !$root_element ) { + my $missed; + foreach my $key ( keys %{$json} ) { + if ( exists $json->{$key}->{__compat} ) { + my $full_key = join q[.], @parts, $key; + $full_key =~ s/^javascript[.]builtins[.]//smx; + $full_key =~ s/^api[.]//smx; + $root_element->{$full_key} = $json->{$key}; + } + else { + $json = $json->{$key}; + push @parts, $key; + last; + } + } + if ($root_element) { + if ($missed) { + Carp::croak("Failed to navigate JSON in $path for key $missed"); + } + } + } + return $root_element; +} + +sub _get_versions { + my ( $object, $mirror, $name ) = @_; + my @versions; + if ($object) { + if ( $object eq 'mirror' ) { + $object = $mirror; + } + if ( ( ref $object ) eq 'HASH' ) { + push @versions, _strip_version($object); + } + else { + push @versions, _strip_version( @{$object} ); + } + } + return @versions; +} + +sub _strip_version { + my (@possible) = @_; + my @approved; + foreach my $version (@possible) { + if ( $version->{version_added} ) { + if ( $version->{version_added} eq 'preview' ) { + next; + } + elsif ( $version->{version_added} =~ + s/^\x{2264}(\d+(?:[.]\d+)?)$/$1/smx ) + { + } + } + push @approved, $version; + } + return @approved; +} + +__END__ +=head1 NAME + +build-bcd-for-firefox - build user agent data from the @mdn/browser-compat-data repo + +=head1 VERSION + +Version 1.55 + +=head1 USAGE + + $ build-bcd-for-firefox + $ build-bcd-for-firefox --path + +=head1 DESCRIPTION + +This program is intended to build a database for the agent method of the Firefox::Marionette class. It builds this database by cloning the @mdn/browser-compat-data repository on github.com and then summarising this data for the agent method. + +The path where the database is stored varies by user and operating system, and can be shown with the --path option. + +=head1 REQUIRED ARGUMENTS + +None + +=head1 OPTIONS + +Option names can be abbreviated to uniqueness and can be stated with singe or double dashes, and option values can be separated from the option name by a space or '=' (as with Getopt::Long). Option names are also case- +sensitive. + +=over 4 + +=item * --help - This page. + +=item * --version - Print the current version of this binary + +=item * --path - Print the path of the local database that is getting built. + +=back + +=head1 CONFIGURATION + +build-bcd-for-firefox requires no configuration files or environment variables. + +=head1 DEPENDENCIES + +build-bcd-for-firefox requires the following non-core Perl modules + +=over + +=item * +L + +=back + +=head1 DIAGNOSTICS + +None. + +=head1 INCOMPATIBILITIES + +None known. + +=head1 EXIT STATUS + +This program will exit with a zero after successfully completing. + +=head1 BUGS AND LIMITATIONS + +To report a bug, or view the current list of bugs, please visit L + +=head1 AUTHOR + +David Dick C<< >> + +=head1 LICENSE AND COPYRIGHT + +Copyright (c) 2024, David Dick C<< >>. All rights reserved. + +This module is free software; you can redistribute it and/or +modify it under the same terms as Perl itself. See L. + +=head1 DISCLAIMER OF WARRANTY + +BECAUSE THIS SOFTWARE IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE SOFTWARE, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE SOFTWARE "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER +EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE +ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE SOFTWARE IS WITH +YOU. SHOULD THE SOFTWARE PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL +NECESSARY SERVICING, REPAIR, OR CORRECTION. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE SOFTWARE AS PERMITTED BY THE ABOVE LICENCE, BE +LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL, +OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE +THE SOFTWARE (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE SOFTWARE TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. diff --git a/lib/Firefox/Marionette.pm b/lib/Firefox/Marionette.pm index eef43dd..d4b0a0a 100644 --- a/lib/Firefox/Marionette.pm +++ b/lib/Firefox/Marionette.pm @@ -47,6 +47,7 @@ use URI(); use URI::Escape(); use Time::HiRes(); use Time::Local(); +use File::HomeDir(); use File::Temp(); use File::stat(); use File::Spec::Unix(); @@ -145,6 +146,16 @@ sub _WATERFOX_CLASSIC_VERSION_EQUIV { return 56; } # https://github.com/MrAlex94/Waterfox/wiki/Versioning-Guidelines +sub BCD_PATH { + my ($create) = @_; + my $directory = File::HomeDir->my_dist_data( 'Firefox-Marionette', + { defined $create && $create == 1 ? ( create => 1 ) : () } ); + if ( defined $directory ) { + return File::Spec->catfile( $directory, 'bcd.json' ); + } + else { return } +} + my $proxy_name_regex = qr/perl_ff_m_\w+/smx; my $tmp_name_regex = qr/firefox_marionette_(?:remote|local)_\w+/smx; my @sig_nums = split q[ ], $Config{sig_num}; @@ -511,7 +522,7 @@ sub _parse_user_agent { # https://developer.mozilla.org/en-US/docs/Web/API/Navigator/userAgent#value if ( !defined $user_agent ) { - $user_agent = $self->{original_agent}; + $user_agent = $self->_original_agent(); } if ( $user_agent =~ m{^ @@ -543,11 +554,19 @@ sub _parse_user_agent { $vendor = 'Apple Computer, Inc.'; $vendor_sub = q[]; } - if ( $user_agent =~ /Win(?:32|64)/smx ) { + elsif ( $self->_is_trident_user_agent($user_agent) ) { + $app_version = $webkit_app; + $vendor = q[]; + $vendor_sub = undef; + } + if ( $user_agent =~ /Win(?:32|64|dows)/smx ) { $platform = 'Win32'; if ( $self->_is_chrome_user_agent($user_agent) ) { $oscpu = undef; } + elsif ( $self->_is_trident_user_agent($user_agent) ) { + $oscpu = undef; + } else { $oscpu = $platform; } @@ -589,7 +608,8 @@ m/^$general_token_re$platform_re$gecko_version_re$gecko_trail_re$firefox_version ( $os_string, $rv_version, $firefox_version ) = ( $1, $2, $3 ); } else { - Firefox::Marionette::Exception->throw('Failed to parse user agent'); + Firefox::Marionette::Exception->throw( + 'Failed to parse user agent:' . $original_string ); } return ( $os_string, $rv_version, $firefox_version ); } @@ -705,7 +725,7 @@ sub agent { my $pref_name = 'general.useragent.override'; my $old_agent = $self->script( $self->_compress_script('return navigator.userAgent') ); - if ( !defined $self->{original_agent} ) { + if ( !defined $self->_original_agent() ) { $self->{original_agent} = $old_agent; } if ( ( scalar @new_list ) > 0 ) { @@ -762,12 +782,12 @@ sub agent { $self->set_pref( 'privacy.donottrackheader.enabled', $false ) ; # trying to blend in with the most common options if ( $self->{stealth} ) { + my %agent_parameters = ( from => $old_agent, to => $user_agent ); $self->script( $self->_compress_script( - <<'_JS_' . Firefox::Marionette::Extension::Stealth->user_agent_contents() ), args => [ $user_agent, $app_version, $platform, $vendor, $vendor_sub, $oscpu ] ); + <<'_JS_' . Firefox::Marionette::Extension::Stealth->user_agent_contents(%agent_parameters) ), args => [ $user_agent, $app_version, $platform, $vendor, $vendor_sub, $oscpu ] ); { let navProto = Object.getPrototypeOf(window.navigator); - let winProto = Object.getPrototypeOf(window); Object.defineProperty(navProto, 'userAgent', {value: arguments[0], writable: true}); Object.defineProperty(navProto, 'appVersion', {value: arguments[1], writable: true}); Object.defineProperty(navProto, 'platform', {value: arguments[2], writable: true}); @@ -776,6 +796,11 @@ sub agent { Object.defineProperty(navProto, 'oscpu', {value: arguments[5], writable: true}); } _JS_ + $self->uninstall( delete $self->{stealth_extension} ); + my $zip = Firefox::Marionette::Extension::Stealth->new( + $self->_original_agent(), $user_agent ); + $self->{stealth_extension} = + $self->_install_extension_by_handle( $zip, 'stealth-0.0.1.xpi' ); } } @@ -4219,21 +4244,33 @@ sub _install_extension { close $handle or Firefox::Marionette::Exception->throw( "Failed to close '$path':$EXTENDED_OS_ERROR"); - $self->install( $path, 1 ); - return; + return $self->install( $path, 1 ); } sub _install_extension_by_handle { - my ( $self, $module, $name ) = @_; + my ( $self, $zip, $name ) = @_; $self->_build_local_extension_directory(); - my $zip = $module->new(); my $path = File::Spec->catfile( $self->{_local_extension_directory}, $name ); - $zip->writeToFileNamed($path) == Archive::Zip::AZ_OK() + unlink $path + or ( $OS_ERROR == POSIX::ENOENT() ) + or Firefox::Marionette::Exception->throw( + "Failed to unlink '$path':$EXTENDED_OS_ERROR"); + my $handle = FileHandle->new( + $path, + Fcntl::O_WRONLY() | Fcntl::O_CREAT() | Fcntl::O_EXCL(), + Fcntl::S_IRUSR() | Fcntl::S_IWUSR() + ) + or Firefox::Marionette::Exception->throw( + "Failed to open '$path' for writing:$EXTENDED_OS_ERROR"); + binmode $handle; + $zip->writeToFileHandle( $handle, 1 ) == Archive::Zip::AZ_OK() or Firefox::Marionette::Exception->throw( "Failed to write to '$path':$EXTENDED_OS_ERROR"); - $self->install( $path, 1 ); - return; + close $handle + or Firefox::Marionette::Exception->throw( + "Failed to close '$path':$EXTENDED_OS_ERROR"); + return $self->install( $path, 1 ); } sub _post_launch_checks_and_setup { @@ -4243,9 +4280,15 @@ sub _post_launch_checks_and_setup { $self->timeouts($timeouts); } if ( $self->{stealth} ) { - $self->_install_extension_by_handle( - 'Firefox::Marionette::Extension::Stealth', - 'stealth-0.0.1.xpi' ); + my $old_user_agent = $self->agent(); + my $zip = Firefox::Marionette::Extension::Stealth->new($old_user_agent); + $self->{stealth_extension} = + $self->_install_extension_by_handle( $zip, 'stealth-0.0.1.xpi' ); + $self->script( + $self->_compress_script( + Firefox::Marionette::Extension::Stealth->user_agent_contents() + ) + ); } if ( $self->{_har} ) { $self->_install_extension( @@ -12136,6 +12179,12 @@ Version 1.55 This is a client module to automate the Mozilla Firefox browser via the L +=head1 CONSTANTS + +=head2 BCD_PATH + +returns the local path used for storing the brower compability data for the L method when the stealth parameter is supplied to the L method. This database is built by the build-bcd-for-firefox binary. + =head1 SUBROUTINES/METHODS =head2 accept_alert @@ -12459,7 +12508,7 @@ These parameters can be used to set a user agent string like so; # user agent is now equal to # Mozilla/5.0 (X11; Linux s390x; rv:109.0) Gecko/20100101 Firefox/115.0 -If the C parameter has supplied to the L method, it will also attempt to change a number of javascript attributes to match the desired browser. The following websites have been very useful in testing these ideas; +If the C parameter has supplied to the L method, it will also attempt to delete/provide dummy implementations for number of L to match the desired browser. The following websites have been very useful in testing these ideas; =over 4 @@ -12469,8 +12518,12 @@ If the C parameter has supplied to the L method, it will also =item * L +=item * L + =back +Importantly, this will break L for any website that relies on it. + See L a discussion of these types of techniques. These changes are not foolproof, but it is interesting to see what can be done with modern browsers. All this behaviour should be regarded as extremely experimental and subject to change. Feedback welcome. =head2 alert_text @@ -14815,7 +14868,7 @@ This list of methods may grow. Marionette L allows web sites to detect that the browser is being automated. Firefox L allows you to disable this functionality while you are automating the browser, but this can be overridden with the C parameter for the L method. This is extremely experimental and feedback is welcome. -If the web site you are trying to automate mysteriously fails when you are automating a workflow, but it works when you perform the workflow manually, you may be dealing with a web site that is hostile to automation. +If the web site you are trying to automate mysteriously fails when you are automating a workflow, but it works when you perform the workflow manually, you may be dealing with a web site that is hostile to automation. I would be very interested if you can supply a test case. At the very least, under these circumstances, it would be a good idea to be aware that there's an L, and potential L in this area. diff --git a/lib/Firefox/Marionette/Extension/Stealth.pm b/lib/Firefox/Marionette/Extension/Stealth.pm index 6c77bb1..72091bf 100644 --- a/lib/Firefox/Marionette/Extension/Stealth.pm +++ b/lib/Firefox/Marionette/Extension/Stealth.pm @@ -1,24 +1,65 @@ package Firefox::Marionette::Extension::Stealth; +use File::HomeDir(); +use File::Spec(); use Archive::Zip(); +use English qw( -no_match_vars ); use strict; use warnings; our $VERSION = '1.55'; -my $inject_name = 'inject.js'; +sub _BUFFER_SIZE { return 65_536 } + my $content_name = 'content.js'; +my %_function_bodies = ( + 'Navigator.bluetooth' => +q[return { getAvailability: function () { return new Promise((resolve, reject) => resolve(false))} };], + 'Navigator.canShare' => q[return true], + 'Navigator.clearAppBadge' => + q[return new Promise((resolve, reject) => resolve(undefined))], + 'Navigator.connection' => +q[let downlink = (new Array(7.55, 1.6))[Math.floor(Math.random() * 2)]; let rtt = (new Array(0, 50, 100))[Math.floor(Math.random() * 3)]; let obj = { onchange: null, effectiveType: decodeURIComponent(\\x27%274g%27\\x27), rtt: rtt, downlink: downlink, saveData: false }; return new NetworkInformation(obj)], + 'Navigator.deprecatedReplaceInURN' => +q[throw TypeError(decodeURIComponent(\\x27Failed to execute %27deprecatedReplaceInURN%27 on %27Navigator%27: Passed URL must be a valid URN URL.\\x27))], + 'Navigator.deprecatedURNtoURL' => +q[throw TypeError(decodeURIComponent(\\x27Failed to execute %27deprecatedURNtoURL%27 on %27Navigator%27: Passed URL must be a valid URN URL.\\x27))], + 'Navigator.deviceMemory' => + q[return (navigator.hardwareConcurrency < 4 ? 4 : 8)], + 'Navigator.getBattery' => +q[return new Promise((resolve, reject) => resolve({ charging: true, chargingTime: 0, dischargingTime: Infinity, level: 1, onchargingchange: null }))], + 'Navigator.getGamePads' => q[return new Array( null, null, null, null )], + 'Navigator.getInstalledRelatedApps' => + q[return new Promise((resolve,reject) => resolve([]))], + 'Navigator.getUserMedia' => q[return getUserMedia], + 'Navigator.gpu' => q[return { wgslLanguageFeatures: { size: 0 } }], + 'Navigator.hid' => +q[return { getDevices: function() { return new Promise((resolve,reject) => resolve([])) }, requestDevices: function() { return new Promise((resolve, reject) => resolve([])) } }], + 'Navigator.ink' => q[return {}], + 'Navigator.keyboard' => +q[return { getLayoutMap: function() { }, lock: function() { }, unlock: function() { } }], + 'Navigator.locks' => + q[return { query: function() { }, request: function() { } }], + 'Navigator.login' => + q[return { setStatus: function() { return undefined } }], + 'Navigator.webkitGetUserMedia' => q[return getUserMedia], + 'Navigator.xr' => q[return new XRSystem({ondevicechange: null})], +); + sub new { - my ($class) = @_; + my ( $class, $from_user_agent_string, $to_user_agent_string ) = @_; my $zip = Archive::Zip->new(); my $manifest = $zip->addString( $class->_manifest_contents(), 'manifest.json' ); $manifest->desiredCompressionMethod( Archive::Zip::COMPRESSION_DEFLATED() ); - my $content = $zip->addString( $class->_content_contents(), $content_name ); + my $content = $zip->addString( + $class->_content_contents( + $from_user_agent_string, $to_user_agent_string + ), + $content_name + ); $content->desiredCompressionMethod( Archive::Zip::COMPRESSION_DEFLATED() ); - my $inject = $zip->addString( $class->_inject_contents(), $inject_name ); - $inject->desiredCompressionMethod( Archive::Zip::COMPRESSION_DEFLATED() ); return $zip; } @@ -47,132 +88,577 @@ _JS_ } sub _content_contents { - my ($class) = @_; + my ( $class, $from_user_agent_string, $to_user_agent_string ) = @_; + my $user_agent_contents = $class->user_agent_contents( + from => $from_user_agent_string, + to => $to_user_agent_string + ); + $user_agent_contents =~ s/\s+/ /smxg; + $user_agent_contents =~ s/\\n/\\\\n/smxg; return <<"_JS_"; { let script = document.createElement('script'); - script.src = chrome.runtime.getURL('$inject_name'); - script.onload = function() { this.remove(); }; + let text = document.createTextNode('$user_agent_contents'); + script.appendChild(text); (document.head || document.documentElement).appendChild(script); } _JS_ } -sub _inject_contents { - my ($class) = @_; - return <<'_JS_' . $class->user_agent_contents(); -{ - let navProto = Object.getPrototypeOf(window.navigator); - let winProto = Object.getPrototypeOf(window); - Object.defineProperty(navProto, 'webdriver', {value: false, writable: true, enumerable: false}); -} -_JS_ -} - sub user_agent_contents { - my ($class) = @_; - return <<'_JS_'; + my ( $class, %parameters ) = @_; + my ( $definition_name, $function_definition ) = + $class->_get_js_function_definition( 'webdriver', 'return false' ); + my $contents = <<"_JS_"; { + if (("console" in window) && ("log" in window.console)) { + console.log("Loading Firefox::Marionette::Extension::Stealth"); + } let navProto = Object.getPrototypeOf(window.navigator); let winProto = Object.getPrototypeOf(window); - let docProto = Object.getPrototypeOf(window.document); - let bluetooth = { - getAvailability: function () { return new Promise((resolve, reject) => resolve(false))}, - }; + $function_definition + Object.defineProperty(navProto, "webdriver", {get: $definition_name, enumerable: false, configurable: true}); + let getUserMedia = window.navigator.mozGetUserMedia; +_JS_ + my $from_user_agent_string = $parameters{from}; + my $to_user_agent_string = $parameters{to}; + if ( defined $to_user_agent_string ) { + my ( $from_browser_type, $from_browser_version ); + my ( $to_browser_type, $to_browser_version ); + if ( $to_user_agent_string =~ /Chrome\/(\d+)/smx ) { + ( $to_browser_type, $to_browser_version ) = ( 'chrome', $1 ); + $contents .= <<'_JS_'; + Object.defineProperty(navProto, "vendor", {value: "Google Inc.", writable: true}); + Object.defineProperty(navProto, "productSub", {value: "20030107", writable: true}); + delete navProto.oscpu; let chrome = { - webstore: function () { }, - app: function () { }, csi: function () { }, + getVariableName: function () { }, loadTimes: function () { }, + metricsPrivate: { + MetricTypeType: { + HISTOGRAM_LINEAR: "histogram-linear", + HISTOGRAM_LOG: "histogram-log" + }, + getFieldTrial: function () { }, + getHistogram: function () { }, + getVariationParams: function () { }, + recordBoolean: function () { }, + recordCount: function () { }, + recordEnumerationValue: function () { }, + recordLongTime: function () { }, + recordMediumCount: function () { }, + recordMediumTime: function () { }, + recordPercentage: function () { }, + recordSmallCount: function () { }, + recordSparseValue: function () { }, + recordSparseValueWithHashMetricName: function () { }, + recordSparseValueWithPersistentHash: function () { }, + recordTime: function () { }, + recordUserAction: function () { }, + recordValue: function () { } + }, + send: function () { }, + timeTicks: { nowInMicroseconds: function () { } }, + webstore: function () { }, + app: function () { }, runtime: { connect: function() { }, sendMessage: function() { } } }; + Object.defineProperty(winProto, "chrome", {value: chrome, writable: true, enumerable: true}); + let canLoadAdAuctionFencedFrame = function() { return true }; - let canShare = function() { return true }; - let getUserMedia = window.navigator.mozGetUserMedia; - let clearAppBadge = function () { return new Promise((resolve, reject) => resolve(undefined))}; - let clearOriginJoinedAdInterestGroup = function (url) { return new Promise((resolve, reject) => { if (new RegExp(/^https:/).exec(url)) { throw DOMException("Permission to leave interest groups denied.") } else { throw new TypeError("Failed to execute 'clearOriginJoinedAdInterestGroup' on 'Navigator': owner '" + url + "' must be a valid https origin") }})}; - delete window.navigator.mozGetUserMedia; - let getBattery = function () { return new Promise((resolve, reject) => resolve({ charging: true, chargingTime: 0, dischargingTime: Infinity, level: 1, onchargingchange: null }))}; - let downlink = (new Array(7.55, 1.6))[Math.floor(Math.random() * 2)]; - let rtt = (new Array(0, 50, 100))[Math.floor(Math.random() * 3)]; - let connection = { effectiveType: '4g', rtt: rtt, downlink: downlink, saveData: false }; + + Object.defineProperty(navProto, "canLoadAdAuctionFencedFrame", {value: canLoadAdAuctionFencedFrame, writable: true, enumerable: true}); + let createAuctionNonce = function() { return crypto.randomUUID() }; - let deprecatedReplaceInURN = function() { throw TypeError("Failed to execute 'deprecatedReplaceInURN' on 'Navigator': Passed URL must be a valid URN URL.") }; - let deprecatedURNtoURL = function() { throw TypeError("Failed to execute 'deprecatedURNtoURL' on 'Navigator': Passed URL must be a valid URN URL.") }; - let getGamePads = function() { return new Array( null, null, null, null ) }; - let getInstalledRelatedApps = function() { return new Promise((resolve,reject) => resolve([])) }; - let gpu = { wgslLanguageFeatures: { size: 0 } }; - let hid = { getDevices: function() { return new Promise((resolve,reject) => resolve([])) }, requestDevices: function() { return new Promise((resolve, reject) => resolve([])) } }; - let joinAdInterestGroup = function (group) { return new Promise((resolve, reject) => { throw new TypeError("Failed to execute 'joinAdInterestGroup' on 'Navigator': The provided value is not of type 'AuctionAdInterestGroup'.")})}; - let keyboard = { getLayoutMap: function() { }, lock: function() { }, unlock: function() { } }; - let leaveAdInterestGroup = function () { throw new TypeError("Failed to execute 'leaveAdInterestGroup' on 'Navigator': May only leaveAdInterestGroup from an https origin.")}; - let locks = { query: function() { }, request: function() { } }; - let login = { setStatus: function() { return undefined } }; - if (navigator.userAgent.match(/Chrome/)) { - Object.defineProperty(navProto, 'vendor', {value: "Google Inc.", writable: true}); - Object.defineProperty(navProto, 'productSub', {value: "20030107", writable: true}); - Object.defineProperty(navProto, 'getUserMedia', {value: getUserMedia, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'webkitGetUserMedia', {value: getUserMedia, writable: true, enumerable: true}); - try { - Object.defineProperty(navProto, 'bluetooth', {value: bluetooth, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'bluetooth', {value: bluetooth, writable: true, enumerable: true}); - } catch(e) { - console.log("Unable to redefine bluetooth:" + e); - } - Object.defineProperty(winProto, 'chrome', {value: chrome, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'canLoadAdAuctionFencedFrame', {value: canLoadAdAuctionFencedFrame, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'canShare', {value: canShare, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'clearAppBadge', {value: clearAppBadge, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'clearOriginJoinedAdInterestGroup', {value: clearOriginJoinedAdInterestGroup, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'connection', {value: connection, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'createAuctionNonce', {value: createAuctionNonce, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'deprecatedReplaceInURN', {value: deprecatedReplaceInURN, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'deprecatedRunAdAuctionEnforcesKAnonymity', {value: false, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'deprecatedURNtoURL', {value: deprecatedURNtoURL, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'deviceMemory', {value: (navigator.hardwareConcurrency < 4 ? 4 : 8), writable: true, enumerable: true}); - Object.defineProperty(navProto, 'getBattery', {value: getBattery, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'getGamePads', {value: getGamePads, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'getInstalledRelatedApps', {value: getInstalledRelatedApps, writable: true, enumerable: true}); - if (!window.navigator.gpu) { - Object.defineProperty(navProto, 'gpu', {value: gpu, writable: true, enumerable: true}); + Object.defineProperty(navProto, "createAuctionNonce", {value: createAuctionNonce, writable: true, enumerable: true}); + + Object.defineProperty(navProto, "deprecatedRunAdAuctionEnforcesKAnonymity", {value: false, writable: true, enumerable: true}); + + delete window.navigator.mozGetUserMedia; +_JS_ + if ( $to_user_agent_string =~ /Edg(?:[eA]|iOS)?\/(\d+)/smx ) { + ( $to_browser_type, $to_browser_version ) = ( 'edge', $1 ); + } + elsif ( $to_user_agent_string =~ /(?:Opera|Presto|OPR)\/(\d+)/smx ) + { + ( $to_browser_type, $to_browser_version ) = ( 'opera', $1 ); + my ( $scrap_name, $scrap_definition ) = + $class->_get_js_function_definition( 'scrap', 'return null' ); + $contents .= <<"_JS_"; + $scrap_definition + Object.defineProperty(winProto, "g_opr", {value: {scrap: $scrap_name}, enumerable: true, configurable: true}); + Object.defineProperty(winProto, "opr", {value: {}, enumerable: true, configurable: true}); +_JS_ + } + } + elsif ( $to_user_agent_string =~ + /Version\/(\d+)(?:[.]\d+)?[ ].*Safari\/\d+/smx ) + { + ( $to_browser_type, $to_browser_version ) = ( 'safari', $1 ); + $contents .= <<'_JS_'; + Object.defineProperty(navProto, "vendor", {value: "Apple Computer, Inc.", writable: true}); + Object.defineProperty(navProto, "productSub", {value: "20030107", writable: true}); + delete navProto.oscpu; + delete window.navigator.mozGetUserMedia; +_JS_ + } + elsif ( $to_user_agent_string =~ /Trident/smx ) { + $contents .= <<'_JS_'; + let docProto = Object.getPrototypeOf(window.document); + delete navProto.productSub; + delete navProto.vendorSub; + delete navProto.oscpu; + delete window.navigator.mozGetUserMedia; + Object.defineProperty(navProto, "vendor", {value: "", writable: true}); + Object.defineProperty(docProto, "documentMode", {value: true, writable: true, enumerable: true}); + Object.defineProperty(navProto, "msDoNotTrack", {value: "0", writable: true}); + Object.defineProperty(winProto, "msWriteProfilerMark", {value: {}, writable: true}); +_JS_ + } + my $general_token_re = qr/Mozilla\/5[.]0[ ]/smx; + my $platform_etc_re = qr/[(][^)]+[)][ ]/smx; + my $gecko_trail_re = qr/Gecko\/20100101[ ]/smx; + my $firefox_version_re = qr/Firefox\/(\d+)[.]0/smx; + if ( $to_user_agent_string =~ +/^$general_token_re$platform_etc_re$gecko_trail_re$firefox_version_re$/smx + ) + { + ( $to_browser_type, $to_browser_version ) = ( 'firefox', $1 ); + $contents .= <<'_JS_'; + Object.defineProperty(navProto, "vendor", {value: "", writable: true}); + Object.defineProperty(navProto, "productSub", {value: "20100101", writable: true}); +_JS_ + } + else { + $contents .= <<'_JS_'; + delete navProto.buildID; + delete window.InstallTrigger; +_JS_ + } + if ( $from_user_agent_string =~ +/^$general_token_re$platform_etc_re$gecko_trail_re$firefox_version_re$/smx + ) + { + ( $from_browser_type, $from_browser_version ) = ( 'firefox', $1 ); + } + if ( $from_browser_version && $to_browser_version ) { + $contents .= $class->_browser_compat_data( + from_browser_type => $from_browser_type, + from_browser_version => $from_browser_version, + to_browser_type => $to_browser_type, + to_browser_version => $to_browser_version, + filters => $parameters{filters}, + ); + } } - Object.defineProperty(navProto, 'hid', {value: hid, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'ink', {value: {}, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'joinAdInterestGroup', {value: joinAdInterestGroup, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'keyboard', {value: keyboard, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'leaveAdInterestGroup', {value: leaveAdInterestGroup, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'locks', {value: locks, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'login', {value: login, writable: true, enumerable: true}); - delete navProto.oscpu; - } else if (navigator.userAgent.match(/Safari/)) { - Object.defineProperty(navProto, 'vendor', {value: "Apple Computer, Inc.", writable: true}); - Object.defineProperty(navProto, 'productSub', {value: "20030107", writable: true}); - Object.defineProperty(navProto, 'getUserMedia', {value: getUserMedia, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'webkitGetUserMedia', {value: getUserMedia, writable: true, enumerable: true}); - /* no bluetooth for Safari - https://developer.mozilla.org/en-US/docs/Web/API/Bluetooth#browser_compatibility */ - Object.defineProperty(winProto, 'chrome', {value: chrome, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'canLoadAdAuctionFencedFrame', {value: canLoadAdAuctionFencedFrame, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'canShare', {value: canShare, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'clearAppBadge', {value: clearAppBadge, writable: true, enumerable: true}); - Object.defineProperty(navProto, 'clearOriginJoinedAdInterestGroup', {value: clearOriginJoinedAdInterestGroup, writable: true, enumerable: true}); - /* no connection for Safari - https://developer.mozilla.org/en-US/docs/Web/API/Navigator/connection#browser_compatibility */ - /* no deviceMemory for Safari - https://developer.mozilla.org/en-US/docs/Web/API/Navigator/deviceMemory#browser_compatibility */ - /* no getBattery for Safari - https://developer.mozilla.org/en-US/docs/Web/API/Navigator/getBattery#browser_compatibility */ - delete navProto.oscpu; - } else if (navigator.userAgent.match(/Trident/)) { - Object.defineProperty(docProto, 'documentMode', {value: true, writable: true, enumerable: true}); + $contents .= <<'_JS_'; + if (("console" in window) && ("log" in window.console)) { + console.log("Loaded Firefox::Marionette::Extension::Stealth"); } - if (navigator.userAgent.match(/^Mozilla\/5[.]0[ ][(][^)]*?;[ ]rv:\d+[.]0[)][ ]Gecko\/20100101[ ]Firefox\/\d+[.]0/)) { - Object.defineProperty(navProto, 'vendor', {value: "", writable: true}); - Object.defineProperty(navProto, 'productSub', {value: "20100101", writable: true}); +} +_JS_ + return $contents; +} + +sub _check_and_add_class { + my ( $class, $property_name, $to_string_tag_allowed ) = @_; + my $javascript_class = $property_name; + my $win_proto_class = $javascript_class; + $win_proto_class =~ s/^Window[.]/winProto./smx; + + my $contents = <<"_JS_"; + if ("$javascript_class" in window) { } else { - delete navProto.buildID; - delete window.InstallTrigger; + window.$javascript_class = class { + constructor(obj) { + for(let key of Object.keys(obj)) { + Object.defineProperty(this, key, {value: obj[key], enumerable: true, configurable: true}); + } + } +_JS_ + if ($to_string_tag_allowed) { + $contents .= <<"_JS_"; + get [Symbol.toStringTag]() { + return "$javascript_class"; + } +_JS_ + } + $contents .= <<"_JS_"; + }; } +_JS_ + return $contents; } + +my $_function_definition_count = 1; + +sub _get_js_function_definition { + my ( $class, $name, $function_body ) = @_; + $_function_definition_count += 1; + my $actual_name = "fm_def_$_function_definition_count"; + return ( $actual_name, <<"_JS_"); +let $actual_name = new Function("$function_body"); + $actual_name.toString = function fm_def() { return "function ${name}() {\\n [native code]\\n}" }; _JS_ } +sub _check_and_add_function { + my ( $class, $property_name, $proposed_change_properties, $deleted_classes ) + = @_; + my $javascript_class = $property_name; + $javascript_class =~ s/[.][\-_@[:alnum:]]+$//smx; + my $function_name = $property_name; + $function_name =~ s/^.*[.]([\-_@[:alnum:]]+)$/$1/smx; + my $parent_class = $javascript_class; + if ( $javascript_class =~ /^(.*?)[.]/smx ) { + $parent_class = ($1); + } + my $contents = q[]; + if ( !$deleted_classes->{$javascript_class} ) { + my ( $definition_name, $function_definition ) = + $class->_get_js_function_definition( $function_name, + $proposed_change_properties->{function_body} ); + $contents .= <<"_JS_"; + $function_definition + if (winProto.$parent_class && winProto.$javascript_class) { + if ("$function_name" in winProto.$javascript_class) { + } else { + Object.defineProperty(winProto.$javascript_class.prototype, "$function_name", {get: $definition_name, enumerable: true, configurable: true}); + } + } else if (window.$parent_class && window.$javascript_class) { + if (window.$javascript_class.prototype) { + if ("$function_name" in window.$javascript_class.prototype) { + } else { + Object.defineProperty(window.$javascript_class.prototype, "$function_name", {get: $definition_name, enumerable: true, configurable: true}); + } + } else { + if ("$function_name" in window.$javascript_class) { + } else { + Object.defineProperty(window.$javascript_class, "$function_name", {get: $definition_name, enumerable: true, configurable: true}); + } + } +_JS_ + if ( $javascript_class eq 'Navigator' ) { + $contents .= <<"_JS_"; + Object.defineProperty(navProto, "$function_name", {get: $definition_name, enumerable: true, configurable: true}); +_JS_ + } + elsif ( $javascript_class eq 'Document' ) { + $contents .= <<"_JS_"; + Object.defineProperty(window.document, "$function_name", {get: $definition_name, enumerable: true, configurable: true}); +_JS_ + } + $contents .= <<"_JS_"; + } +_JS_ + } + return $contents; +} + +sub _check_and_delete_class { + my ( $class, $property_name ) = @_; + my $parent_class = $property_name; + if ( $property_name =~ /^(.*?)[.]/smx ) { + ($parent_class) = ($1); + } + my $contents = <<"_JS_"; + if (winProto.$parent_class && winProto.$property_name) { + delete winProto.$property_name; + } else if (window.$parent_class && window.$property_name) { + delete window.$property_name; + } +_JS_ + if ( $property_name eq 'SubtleCrypto' ) { + $contents .= <<'_JS_'; + if ("crypto" in window) { + delete window["crypto"]; + } +_JS_ + } + else { + my $lc_property_name = lc $property_name; + $contents .= <<"_JS_"; + if ("$lc_property_name" in window) { + delete window["$lc_property_name"]; + } +_JS_ + } + return $contents; +} + +sub _check_and_delete_function { + my ( $class, $property_name, $deleted_classes ) = @_; + my $contents = q[]; + my $javascript_class = $property_name; + $javascript_class =~ s/[.][\-_@[:alpha:]]+$//smx; + my $function_name = $property_name; + $function_name =~ s/^.*[.]([\-_@[:alpha:]]+)$/$1/smx; + my $parent_class = $javascript_class; + if ( $javascript_class =~ /^(.*?)[.]/smx ) { + $parent_class = ($1); + } + if ( !$deleted_classes->{$javascript_class} ) { + $contents .= <<"_JS_"; + if (winProto.$parent_class && winProto.$javascript_class) { + delete winProto.$javascript_class\["$function_name"\]; + } else if (window.$parent_class && window.$javascript_class) { + if (window.$javascript_class.prototype) { + delete window.$javascript_class.prototype\["$function_name"\]; + } + delete window.$javascript_class\["$function_name"\]; + } +_JS_ + } + if ( $javascript_class eq 'Navigator' ) { + $contents .= <<"_JS_"; + if ("$function_name" in navProto) { + delete navProto["$function_name"]; + } else if ("$function_name" in navigator) { + delete navigator["$function_name"]; + } +_JS_ + } + elsif ( $javascript_class eq 'Document' ) { + $contents .= <<"_JS_"; + if (("document" in winProto) && ("$function_name" in winProto.document)) { + delete winProto.document["$function_name"]; + } else if (("document" in window) && ("$function_name" in window.document)) { + delete window.document["$function_name"]; + } +_JS_ + } + elsif ( $javascript_class eq 'Window' ) { + $contents .= <<"_JS_"; + if ("$function_name" in winProto) { + delete winProto["$function_name"]; + } else if ("$function_name" in window) { + delete window["$function_name"]; + } +_JS_ + } + return $contents; +} + +sub _read_bcd { + my ($class) = @_; + my %browser_properties; + my $bcd_path = Firefox::Marionette::BCD_PATH(); + if ( ( defined $bcd_path ) + && ( my $bcd_handle = FileHandle->new( $bcd_path, Fcntl::O_RDONLY() ) ) + ) + { + my $bcd_contents; + my $result; + while ( $result = $bcd_handle->read( my $buffer, _BUFFER_SIZE() ) ) { + $bcd_contents .= $buffer; + } + close $bcd_handle + or Firefox::Marionette::Exception->throw( + "Failed to close '$bcd_path':$EXTENDED_OS_ERROR"); + %browser_properties = %{ JSON->new()->decode($bcd_contents) }; + } + elsif ( $OS_ERROR == POSIX::ENOENT() ) { + Carp::carp( + q[BCD file is not available. Please run 'build-bcd-for-firefox']); + } + else { + Firefox::Marionette::Exception->throw( + "Failed to open '$bcd_path' for reading:$EXTENDED_OS_ERROR"); + } + return %browser_properties; +} + +sub _available_in { + my ( $class, %properties ) = @_; + my $available; + my $browser_type = $properties{browser_type}; + foreach my $proposed_change_properties ( @{ $properties{changes} } ) { + if ( $proposed_change_properties->{add} ) { + if ( $proposed_change_properties->{add} <= + $properties{browser_version} ) + { + if ( !$proposed_change_properties->{pref_name} ) { + if ( !defined $proposed_change_properties->{function_body} ) + { + $proposed_change_properties->{function_body} = + $_function_bodies{ $properties{property_name} } + || 'return null'; + } + $available = $proposed_change_properties; + } + } + } + else { + if ( $proposed_change_properties->{rm} <= + $properties{browser_version} ) + { + $available = undef; + } + } + } + return $available; +} + +sub _this_change_should_be_processed { + my ( $class, $proposed_change, $property_name, $change_number, $filters ) = + @_; + if ( defined $filters ) { + if ( $property_name !~ /$filters/smx ) { + return 0; + } + } + return 1; +} + +sub _browser_compat_data { + my ( $class, %parameters ) = @_; + my %browser_properties = $class->_read_bcd(); + my $contents = q[]; + my %deleted_classes; + my $change_number = 0; + VERSION: + foreach my $property_name ( sort { $a cmp $b } keys %browser_properties ) { + my $property_object = $browser_properties{$property_name}; + my $property_type = $property_object->{type}; + my %from_properties = ( + browser_type => $parameters{from_browser_type}, + browser_version => $parameters{from_browser_version}, + property_type => $property_type, + property_name => $property_name, + changes => $browser_properties{$property_name}{browsers} + { $parameters{from_browser_type} }, + ); + my %to_properties = ( + browser_type => $parameters{to_browser_type}, + browser_version => $parameters{to_browser_version}, + property_type => $property_type, + property_name => $property_name, + changes => $browser_properties{$property_name}{browsers} + { $parameters{to_browser_type} }, + ); + my ( $delete_property, $add_property, $change_properties ); + if ( my $proposed_change_properties = + $class->_available_in(%from_properties) ) + { + if ( !$class->_available_in(%to_properties) ) { + $delete_property = 1; + } + } + else { + if ( my $proposed_change_properties = + $class->_available_in(%to_properties) ) + { + $add_property = 1; + $change_properties = $proposed_change_properties; + } + } + my $change_details = { + delete_property => $delete_property, + add_property => $add_property, + change_number => $change_number, + property_name => $property_name, + property_type => $property_type, + filters => $parameters{filters}, + proposed_change_properties => $change_properties, + deleted_classes => \%deleted_classes, + to_string_tag_allowed => + $browser_properties{'Symbol.toStringTag'}{browsers} + { $parameters{to_browser_type} }[0]{add} < + $parameters{to_browser_version}, + }; + $contents .= $class->_process_change($change_details); + } + return $contents; +} + +sub _process_change { + my ( $class, $change_details ) = @_; + my $contents = q[]; + if ( $change_details->{delete_property} ) { + if ( $change_details->{property_type} eq 'class' ) { + my $proposed_change = $class->_check_and_delete_class( + $change_details->{property_name} ); + if ( + $class->_this_change_should_be_processed( + $proposed_change, + $change_details->{property_name}, + $change_details->{change_number}, + $change_details->{filters}, + ) + ) + { + $contents .= $proposed_change; + $change_details->{deleted_classes} + ->{ $change_details->{property_name} } = 1; + } + } + else { + if ( + my $proposed_change = $class->_check_and_delete_function( + $change_details->{property_name}, + $change_details->{deleted_classes} + ) + ) + { + if ( + $class->_this_change_should_be_processed( + $proposed_change, + $change_details->{property_name}, + $change_details->{change_number}, + $change_details->{filters}, + ) + ) + { + $contents .= $proposed_change; + } + } + } + } + elsif ( $change_details->{add_property} ) { + if ( $change_details->{property_type} eq 'class' ) { + my $proposed_change = $class->_check_and_add_class( + $change_details->{property_name}, + $change_details->{to_string_tag_allowed} + ); + if ( + $class->_this_change_should_be_processed( + $proposed_change, + $change_details->{property_name}, + $change_details->{change_number}, + $change_details->{filters}, + ) + ) + { + $contents .= $proposed_change; + } + } + else { + if ( + my $proposed_change = $class->_check_and_add_function( + $change_details->{property_name}, + $change_details->{proposed_change_properties}, + $change_details->{deleted_classes} + ) + ) + { + if ( + $class->_this_change_should_be_processed( + $proposed_change, + $change_details->{property_name}, + $change_details->{change_number}, + $change_details->{filters}, + ) + ) + { + $contents .= $proposed_change; + } + } + } + } + return $contents; +} + 1; # Magic true value required at end of module __END__ diff --git a/t/01-marionette.t b/t/01-marionette.t index b8fc98a..fb640a0 100755 --- a/t/01-marionette.t +++ b/t/01-marionette.t @@ -79,7 +79,6 @@ my $useragents_me_uri = qq[data:application/json,{"about": "Use this API to get my $min_geo_version = 60; my $min_stealth_version = 59; -my $min_execute_script_with_null_args_version = 45; if (($^O eq 'MSWin32') || ($^O eq 'cygwin')) { } elsif ($> == 0) { # see RT#131304 @@ -114,40 +113,6 @@ foreach my $sig_name (@sig_names) { $SIG{INT} = sub { $terminated = 1; die "Caught an INT signal"; }; $SIG{TERM} = sub { $terminated = 1; die "Caught a TERM signal"; }; -sub _check_navigator_attributes { - my ($firefox, $major_version, $user_agent, %user_agents_to_js) = @_; - foreach my $key (qw( - platform - appVersion - )) { - my $value = $firefox->script('return navigator.' . $key); - if ($user_agent =~ /^libwww[-]perl/smx) { - ok(defined $value, "navigator.$key is unchanged as '$value'"); - } elsif (defined $user_agents_to_js{$user_agent}{$key}) { - ok($value eq $user_agents_to_js{$user_agent}{$key}, "navigator.$key is now '$user_agents_to_js{$user_agent}{$key}':$value"); - } else { - ok(!defined $value, "navigator.$key is undefined"); - } - } - if ($major_version > $min_stealth_version) { - foreach my $key (qw( - productSub - vendor - vendorSub - oscpu - )) { - my $value = $firefox->script('return navigator.' . $key); - if ($user_agent =~ /^libwww[-]perl/smx) { - ok(defined $value, "navigator.$key is unchanged as '$value'"); - } elsif (defined $user_agents_to_js{$user_agent}{$key}) { - ok($value eq $user_agents_to_js{$user_agent}{$key}, "navigator.$key is now '$user_agents_to_js{$user_agent}{$key}':$value"); - } else { - ok(!defined $value, "navigator.$key is undefined"); - } - } - } -} - sub wait_for_server_on { my ($daemon, $pid) = @_; my $host = URI->new($daemon->url())->host(); @@ -1764,228 +1729,8 @@ SKIP: { ok($hash->{value} == 2, "Value returned from script is the numeric 2 in a hash"); } } - # checking against - # https://browserleaks.com/javascript - # https://www.amiunique.org/fingerprint - # https://bot.sannysoft.com/ my $webdriver = $firefox->script('return navigator.webdriver'); ok(!$webdriver, "navigator.webdriver returns false when stealth is on"); - my $freebsd_118_user_agent_string = 'Mozilla/5.0 (X11; FreeBSD amd64; rv:109.0) Gecko/20100101 Firefox/118.0'; - my %user_agents_to_js = ( - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36' => - { - platform => 'Win32', - appVersion => '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', - productSub => '20030107', - vendor => 'Google Inc.', - vendorSub => '', - oscpu => undef, - }, - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0' => - { - platform => 'Win32', - appVersion => '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0', - productSub => '20030107', - vendor => 'Google Inc.', - vendorSub => '', - oscpu => undef, - }, - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15' => - { - platform => 'MacIntel', - appVersion => '5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15', - productSub => '20030107', - vendor => 'Apple Computer, Inc.', - vendorSub => '', - oscpu => undef, - }, - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:109.0) Gecko/20100101 Firefox/115.0' => - { - platform => 'MacIntel', - appVersion => '5.0 (Macintosh)', - productSub => '20100101', - vendor => '', - vendorSub => '', - oscpu => 'Intel Mac OS X 10.13', - }, - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0' => - { - platform => 'Win32', - appVersion => '5.0 (Windows)', - productSub => '20100101', - vendor => '', - vendorSub => '', - oscpu => 'Win32', - }, - 'Mozilla/5.0 (X11; OpenBSD amd64; rv:109.0) Gecko/20100101 Firefox/109.0' => - { - platform => 'OpenBSD amd64', - appVersion => '5.0 (X11)', - productSub => '20100101', - vendor => '', - vendorSub => '', - oscpu => 'OpenBSD amd64', - }, - 'Mozilla/5.0 (X11; NetBSD amd64; rv:120.0) Gecko/20100101 Firefox/120.0' => - { - platform => 'NetBSD amd64', - appVersion => '5.0 (X11)', - productSub => '20100101', - vendor => '', - vendorSub => '', - oscpu => 'NetBSD amd64', - }, - 'Mozilla/5.0 (X11; Linux s390x; rv:109.0) Gecko/20100101 Firefox/115.0' => - { - platform => 'Linux s390x', - appVersion => '5.0 (X11)', - productSub => '20100101', - vendor => '', - vendorSub => '', - oscpu => 'Linux s390x', - }, - 'Mozilla/5.0 (X11; DragonFly x86_64; rv:108.0) Gecko/20100101 Firefox/108.0' => - { - platform => 'DragonFly x86_64', - appVersion => '5.0 (X11)', - productSub => '20100101', - vendor => '', - vendorSub => '', - oscpu => 'DragonFly x86_64', - }, - $freebsd_118_user_agent_string => - { - platform => 'FreeBSD amd64', - appVersion => '5.0 (X11)', - productSub => '20100101', - vendor => '', - vendorSub => '', - oscpu => 'FreeBSD amd64', - }, - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0' => - { - platform => 'Win32', - appVersion => '5.0 (Windows)', - productSub => '20100101', - vendor => '', - vendorSub => '', - oscpu => 'Win32', - }, - 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_7_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1', - { - platform => 'iPhone', - appVersion => '5.0 (iPhone; CPU iPhone OS 16_7_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1', - productSub => '20030107', - vendor => 'Apple Computer, Inc.', - vendorSub => '', - oscpu => undef, - }, - 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Mobile Safari/537.36', - { - platform => 'Linux armv81', - appVersion => '5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Mobile Safari/537.36', - productSub => '20030107', - vendor => 'Google Inc.', - vendorSub => '', - oscpu => undef, - }, - 'libwww-perl/6.72' => { }, - ); - foreach my $user_agent (sort { $a cmp $b } keys %user_agents_to_js) { - if (($major_version < $min_execute_script_with_null_args_version) && (exists $user_agents_to_js{$user_agent}{oscpu}) && (!defined $user_agents_to_js{$user_agent}{oscpu})) { - diag("Skipping '$user_agent' as oscpu will be null and executeScript cannot handle null arguments for older firefoxen"); - next; - } - ok($user_agent, "Testing '$user_agent'"); - ok($firefox->agent($user_agent), "\$firefox->agent(\"$user_agent\") succeeded"); - if ($major_version > $min_stealth_version) { - _check_navigator_attributes($firefox, $major_version, $user_agent, %user_agents_to_js); - } - ok($firefox->go("about:blank"), "\$firefox->go(\"about:blank\") loaded successfully for user agent test of values"); - _check_navigator_attributes($firefox, $major_version, $user_agent, %user_agents_to_js); - } - if ($major_version > $min_stealth_version) { - my $agent = $firefox->agent(undef); - ok($agent, - "\$firefox->agent(undef) should return 'libwww-perl/6.72'"); - ok($agent eq 'libwww-perl/6.72', - "\$firefox->agent(undef) did return '$agent'"); - $firefox->set_javascript(0); - ok(!$firefox->get_pref('javascript.enabled'), "Javascript is disabled for $agent"); - $firefox->set_javascript(undef); - ok($firefox->get_pref('javascript.enabled'), "Javascript is enabled for $agent"); - $firefox->set_javascript(0); - ok(!$firefox->get_pref('javascript.enabled'), "Javascript is disabled for $agent"); - $firefox->set_javascript(1); - ok($firefox->get_pref('javascript.enabled'), "Javascript is enabled for $agent"); - $agent = $firefox->agent(version => 120); - ok($agent eq $original_agent, - "\$firefox->agent(version => 120) should return '$original_agent'"); - ok($agent eq $original_agent, - "\$firefox->agent(version => 120) did return '$agent'"); - $agent = $firefox->agent(increment => -5); - my $correct_agent = $original_agent; - $correct_agent =~ s/rv:\d+/rv:120/smx; - $correct_agent =~ s/Firefox\/\d+/Firefox\/120/smx; - ok($correct_agent, - "\$firefox->agent(increment => -5) should return '$correct_agent'"); - ok($agent eq $correct_agent, - "\$firefox->agent(increment => -5) did return '$agent'"); - $agent = $firefox->agent(version => 108); - $correct_agent = $original_agent; - my $increment_major_version = $major_version - 5; - my $increment_rv_version = $increment_major_version < 120 && $increment_major_version > 109 ? 109 : $increment_major_version; - $correct_agent =~ s/rv:\d+/rv:$increment_rv_version/smx; - $correct_agent =~ s/Firefox\/\d+/Firefox\/$increment_major_version/smx; - ok($agent, - "\$firefox->agent(version => 108) should return '$correct_agent'"); - ok($agent eq $correct_agent, - "\$firefox->agent(version => 108) did return '$agent'"); - $agent = $firefox->agent(undef); - $correct_agent = $original_agent; - $correct_agent =~ s/rv:\d+/rv:108/smx; - $correct_agent =~ s/Firefox\/\d+/Firefox\/108/smx; - ok($agent, - "\$firefox->agent(undef) should return '$correct_agent'"); - ok($agent eq $correct_agent, - "\$firefox->agent(undef) did return '$agent'"); - $firefox->agent(os => 'Win64'); - $agent = $firefox->agent(undef); - ok($agent =~ /^Mozilla\/5[.]0[ ][(]Windows[ ]NT[ ]10[.]0;[ ]Win64;[ ]x64;[ ]rv:\d{2,3}[.]0[)][ ]Gecko\/20100101[ ]Firefox\/\d{2,3}[.]0$/smx, - "\$firefox->agent(os => 'Win64') did return '$agent'"); - $firefox->agent(os => 'FreeBSD', version => 110); - $agent = $firefox->agent(undef); - ok($agent =~ /^Mozilla\/5[.]0[ ][(]X11;[ ]FreeBSD[ ]amd64;[ ]rv:109.0[)][ ]Gecko\/20100101[ ]Firefox\/110.0$/smx, - "\$firefox->agent(os => 'FreeBSD', version => 110) did return '$agent'"); - $firefox->agent(os => 'linux', arch => 'i686'); - $agent = $firefox->agent(undef); - ok($agent =~ /^Mozilla\/5[.]0[ ][(]X11;[ ]Linux[ ]i686;[ ]rv:\d{2,3}.0[)][ ]Gecko\/20100101[ ]Firefox\/\d{2,3}.0$/smx, - "\$firefox->agent(os => 'linux', arch => 'i686') did return '$agent'"); - $firefox->agent(os => 'darwin'); - $agent = $firefox->agent(undef); - ok($agent =~ /^Mozilla\/5[.]0[ ][(]Macintosh;[ ]Intel[ ]Mac[ ]OS[ ]X[ ]\d+[.]\d+;[ ]rv:\d{2,3}.0[)][ ]Gecko\/20100101[ ]Firefox\/\d{2,3}.0$/smx, - "\$firefox->agent(os => 'darwin') did return '$agent'"); - $firefox->agent(os => 'darwin', platform => 'X11'); - $agent = $firefox->agent(undef); - ok($agent =~ /^Mozilla\/5[.]0[ ][(]X11;[ ]Intel[ ]Mac[ ]OS[ ]X[ ]\d+[.]\d+;[ ]rv:\d{2,3}.0[)][ ]Gecko\/20100101[ ]Firefox\/\d{2,3}.0$/smx, - "\$firefox->agent(os => 'darwin', platform => 'X11') did return '$agent'"); - $firefox->agent(os => 'darwin', arch => '10.13'); - $agent = $firefox->agent(undef); - ok($agent =~ /^Mozilla\/5[.]0[ ][(]Macintosh;[ ]Intel[ ]Mac[ ]OS[ ]X[ ]10[.]13;[ ]rv:\d{2,3}.0[)][ ]Gecko\/20100101[ ]Firefox\/\d{2,3}.0$/smx, - "\$firefox->agent(os => 'darwin', arch => '10.13') did return '$agent'"); - $firefox->agent(os => 'freebsd', version => 118); - $agent = $firefox->agent(undef); - ok($agent eq $freebsd_118_user_agent_string, - "\$firefox->agent(os => 'freebsd', version => '118') did return '$agent'"); - eval { $firefox->agent(version => 'blah') }; - my $exception = $@; - chomp $exception; - ok(ref $@ eq 'Firefox::Marionette::Exception', "\$firefox->agent(version => 'blah') throws an exception:$exception"); - eval { $firefox->agent(increment => 'blah') }; - $exception = $@; - chomp $exception; - ok(ref $@ eq 'Firefox::Marionette::Exception', "\$firefox->agent(increment => 'blah') throws an exception:$exception"); - } if (($tls_tests_ok) && ($ENV{RELEASE_TESTING})) { my $json; if ($major_version < 50) { diff --git a/t/04-browserfeatcl.t b/t/04-browserfeatcl.t new file mode 100644 index 0000000..da0272b --- /dev/null +++ b/t/04-browserfeatcl.t @@ -0,0 +1,497 @@ +#! /usr/bin/perl -w + +use strict; +use Firefox::Marionette(); +use Test::More; +use File::Spec(); +use MIME::Base64(); +use Socket(); +use Config; +use Crypt::URandom(); +use lib qw(t/); + +$SIG{INT} = sub { die "Caught an INT signal"; }; +$SIG{TERM} = sub { die "Caught a TERM signal"; }; + +my $min_stealth_version = 59; +my $min_execute_script_with_null_args_version = 45; + +SKIP: { + if (!$ENV{RELEASE_TESTING}) { + plan skip_all => "Author tests not required for installation"; + } + if ($^O eq 'MSWin32') { + plan skip_all => "Cannot test in a $^O environment"; + } else { + delete @ENV{qw(IFS CDPATH ENV BASH_ENV)}; + if (defined $ENV{PATH}) { + $ENV{PATH} = '/usr/local/sbin:/usr/sbin:/usr/local/bin:/usr/bin:/bin'; + if ($^O eq 'netbsd') { + $ENV{PATH} .= ":/usr/pkg/sbin:/usr/pkg/bin"; + } + } + } + require test_daemons; + if (!Test::Daemon::Nginx->available()) { + plan skip_all => "nginx does not appear to be available"; + } + if ((Firefox::Marionette::BCD_PATH()) && (-f Firefox::Marionette::BCD_PATH())) { + } else { + plan skip_all => "BCD does not appear to be available. Please run build-bcd-for-firefox."; + } + my $nginx_listen = '127.0.0.1'; + my $htdocs = File::Spec->catdir(Cwd::cwd(), 'browserfeatcl'); + my $nginx = Test::Daemon::Nginx->new(listen => $nginx_listen, htdocs => $htdocs, index => 'index.html'); + ok($nginx, "Started nginx Server on $nginx_listen on port " . $nginx->port() . ", with pid " . $nginx->pid()); + $nginx->wait_until_port_open(); + my $debug = $ENV{FIREFOX_DEBUG} || 0; + my $visible = $ENV{FIREFOX_VISIBLE} || 0; + my %extra_parameters; + if ($ENV{FIREFOX_BINARY}) { + $extra_parameters{binary} = $ENV{FIREFOX_BINARY}; + } + my $firefox = Firefox::Marionette->new( + %extra_parameters, + debug => $debug, + visible => $visible, + ); + ok($firefox, "Created a normal firefox object"); + my ($major_version, $minor_version, $patch_version) = split /[.]/smx, $firefox->browser_version(); + my $original_agent = $firefox->agent(); + ok($firefox->script('return navigator.webdriver') == JSON::true(), "\$firefox->script('return navigator.webdriver') returns true"); + my $webdriver_definition_script = 'let descriptor = Object.getOwnPropertyDescriptor(Object.getPrototypeOf(navigator), "webdriver"); return descriptor.get.toString();'; + my $original_webdriver_definition = $firefox->script($webdriver_definition_script); + my $quoted_webdriver_definition = $original_webdriver_definition; + $quoted_webdriver_definition =~ s/\n/\\n/smxg; + my $webdriver_def_regex = qr/function[ ]webdriver[(][)][ ][{]\n[ ]+\[native[ ]code\]\n[}]/smx; + ok($original_webdriver_definition =~ /^$webdriver_def_regex$/smx, "Webdriver definition matches regex:$quoted_webdriver_definition"); + ok($firefox->quit() == 0, "\$firefox->quit() succeeded"); + $firefox = Firefox::Marionette->new( + %extra_parameters, + debug => $debug, + visible => $visible, + stealth => 1, + devtools => 1, + ); + ok($firefox, "Created a stealth firefox object"); + # checking against + # https://browserleaks.com/javascript + # https://www.amiunique.org/fingerprint + # https://bot.sannysoft.com/ + my $freebsd_118_user_agent_string = 'Mozilla/5.0 (X11; FreeBSD amd64; rv:109.0) Gecko/20100101 Firefox/118.0'; + my %user_agents_to_js = ( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36' => + { + platform => 'Win32', + appVersion => '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + productSub => '20030107', + vendor => 'Google Inc.', + vendorSub => '', + oscpu => undef, + }, + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0' => + { + platform => 'Win32', + appVersion => '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0', + productSub => '20030107', + vendor => 'Google Inc.', + vendorSub => '', + oscpu => undef, + }, + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/109.0.0.0', + { + platform => 'Linux x86_64', + appVersion => '5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/109.0.0.0', + productSub => '20030107', + vendor => 'Google Inc.', + vendorSub => '', + oscpu => undef, + }, + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15' => + { + platform => 'MacIntel', + appVersion => '5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15', + productSub => '20030107', + vendor => 'Apple Computer, Inc.', + vendorSub => '', + oscpu => undef, + }, + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:109.0) Gecko/20100101 Firefox/115.0' => + { + platform => 'MacIntel', + appVersion => '5.0 (Macintosh)', + productSub => '20100101', + vendor => '', + vendorSub => '', + oscpu => 'Intel Mac OS X 10.13', + }, + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.0' => + { + platform => 'Win32', + appVersion => '5.0 (Windows)', + productSub => '20100101', + vendor => '', + vendorSub => '', + oscpu => 'Win32', + }, + 'Mozilla/5.0 (X11; OpenBSD amd64; rv:109.0) Gecko/20100101 Firefox/109.0' => + { + platform => 'OpenBSD amd64', + appVersion => '5.0 (X11)', + productSub => '20100101', + vendor => '', + vendorSub => '', + oscpu => 'OpenBSD amd64', + }, + 'Mozilla/5.0 (X11; NetBSD amd64; rv:120.0) Gecko/20100101 Firefox/120.0' => + { + platform => 'NetBSD amd64', + appVersion => '5.0 (X11)', + productSub => '20100101', + vendor => '', + vendorSub => '', + oscpu => 'NetBSD amd64', + }, + 'Mozilla/5.0 (X11; Linux s390x; rv:109.0) Gecko/20100101 Firefox/115.0' => + { + platform => 'Linux s390x', + appVersion => '5.0 (X11)', + productSub => '20100101', + vendor => '', + vendorSub => '', + oscpu => 'Linux s390x', + }, + 'Mozilla/5.0 (X11; DragonFly x86_64; rv:108.0) Gecko/20100101 Firefox/108.0' => + { + platform => 'DragonFly x86_64', + appVersion => '5.0 (X11)', + productSub => '20100101', + vendor => '', + vendorSub => '', + oscpu => 'DragonFly x86_64', + }, + $freebsd_118_user_agent_string => + { + platform => 'FreeBSD amd64', + appVersion => '5.0 (X11)', + productSub => '20100101', + vendor => '', + vendorSub => '', + oscpu => 'FreeBSD amd64', + }, + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0' => + { + platform => 'Win32', + appVersion => '5.0 (Windows)', + productSub => '20100101', + vendor => '', + vendorSub => '', + oscpu => 'Win32', + }, + 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_7_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1', + { + platform => 'iPhone', + appVersion => '5.0 (iPhone; CPU iPhone OS 16_7_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1', + productSub => '20030107', + vendor => 'Apple Computer, Inc.', + vendorSub => '', + oscpu => undef, + }, + 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Mobile Safari/537.36', + { + platform => 'Linux armv81', + appVersion => '5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Mobile Safari/537.36', + productSub => '20030107', + vendor => 'Google Inc.', + vendorSub => '', + oscpu => undef, + }, + 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko', + { + platform => 'Win32', + appVersion => '5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko', + productSub => undef, + vendor => '', + vendorSub => undef, + oscpu => undef, + }, + 'libwww-perl/6.72' => { }, + ); + foreach my $user_agent (sort { $a cmp $b } keys %user_agents_to_js) { + if (($major_version < $min_execute_script_with_null_args_version) && (exists $user_agents_to_js{$user_agent}{oscpu}) && (!defined $user_agents_to_js{$user_agent}{oscpu})) { + diag("Skipping '$user_agent' as oscpu will be null and executeScript cannot handle null arguments for older firefoxen"); + next; + } + ok($user_agent, "Testing '$user_agent'"); + ok($firefox->agent($user_agent), "\$firefox->agent(\"$user_agent\") succeeded"); + if ($major_version > $min_stealth_version) { + _check_navigator_attributes($firefox, $major_version, $user_agent, %user_agents_to_js); + } + ok($firefox->go("about:blank"), "\$firefox->go(\"about:blank\") loaded successfully for user agent test of values"); + _check_navigator_attributes($firefox, $major_version, $user_agent, %user_agents_to_js); + } + if ($major_version > $min_stealth_version) { + my $agent = $firefox->agent(undef); + ok($agent, + "\$firefox->agent(undef) should return 'libwww-perl/6.72'"); + ok($agent eq 'libwww-perl/6.72', + "\$firefox->agent(undef) did return '$agent'"); + $firefox->set_javascript(0); + ok(!$firefox->get_pref('javascript.enabled'), "Javascript is disabled for $agent"); + $firefox->set_javascript(undef); + ok($firefox->get_pref('javascript.enabled'), "Javascript is enabled for $agent"); + $firefox->set_javascript(0); + ok(!$firefox->get_pref('javascript.enabled'), "Javascript is disabled for $agent"); + $firefox->set_javascript(1); + ok($firefox->get_pref('javascript.enabled'), "Javascript is enabled for $agent"); + $agent = $firefox->agent(version => 120); + ok($agent eq $original_agent, + "\$firefox->agent(version => 120) should return '$original_agent'"); + ok($agent eq $original_agent, + "\$firefox->agent(version => 120) did return '$agent'"); + $agent = $firefox->agent(increment => -5); + my $correct_agent = $original_agent; + $correct_agent =~ s/rv:\d+/rv:120/smx; + $correct_agent =~ s/Firefox\/\d+/Firefox\/120/smx; + ok($correct_agent, + "\$firefox->agent(increment => -5) should return '$correct_agent'"); + ok($agent eq $correct_agent, + "\$firefox->agent(increment => -5) did return '$agent'"); + $agent = $firefox->agent(version => 108); + $correct_agent = $original_agent; + my $increment_major_version = $major_version - 5; + my $increment_rv_version = $increment_major_version < 120 && $increment_major_version > 109 ? 109 : $increment_major_version; + $correct_agent =~ s/rv:\d+/rv:$increment_rv_version/smx; + $correct_agent =~ s/Firefox\/\d+/Firefox\/$increment_major_version/smx; + ok($agent, + "\$firefox->agent(version => 108) should return '$correct_agent'"); + ok($agent eq $correct_agent, + "\$firefox->agent(version => 108) did return '$agent'"); + $agent = $firefox->agent(undef); + $correct_agent = $original_agent; + $correct_agent =~ s/rv:\d+/rv:108/smx; + $correct_agent =~ s/Firefox\/\d+/Firefox\/108/smx; + ok($agent, + "\$firefox->agent(undef) should return '$correct_agent'"); + ok($agent eq $correct_agent, + "\$firefox->agent(undef) did return '$agent'"); + $firefox->agent(os => 'Win64'); + $agent = $firefox->agent(undef); + ok($agent =~ /^Mozilla\/5[.]0[ ][(]Windows[ ]NT[ ]10[.]0;[ ]Win64;[ ]x64;[ ]rv:\d{2,3}[.]0[)][ ]Gecko\/20100101[ ]Firefox\/\d{2,3}[.]0$/smx, + "\$firefox->agent(os => 'Win64') did return '$agent'"); + $firefox->agent(os => 'FreeBSD', version => 110); + $agent = $firefox->agent(undef); + ok($agent =~ /^Mozilla\/5[.]0[ ][(]X11;[ ]FreeBSD[ ]amd64;[ ]rv:109.0[)][ ]Gecko\/20100101[ ]Firefox\/110.0$/smx, + "\$firefox->agent(os => 'FreeBSD', version => 110) did return '$agent'"); + $firefox->agent(os => 'linux', arch => 'i686'); + $agent = $firefox->agent(undef); + ok($agent =~ /^Mozilla\/5[.]0[ ][(]X11;[ ]Linux[ ]i686;[ ]rv:\d{2,3}.0[)][ ]Gecko\/20100101[ ]Firefox\/\d{2,3}.0$/smx, + "\$firefox->agent(os => 'linux', arch => 'i686') did return '$agent'"); + $firefox->agent(os => 'darwin'); + $agent = $firefox->agent(undef); + ok($agent =~ /^Mozilla\/5[.]0[ ][(]Macintosh;[ ]Intel[ ]Mac[ ]OS[ ]X[ ]\d+[.]\d+;[ ]rv:\d{2,3}.0[)][ ]Gecko\/20100101[ ]Firefox\/\d{2,3}.0$/smx, + "\$firefox->agent(os => 'darwin') did return '$agent'"); + $firefox->agent(os => 'darwin', platform => 'X11'); + $agent = $firefox->agent(undef); + ok($agent =~ /^Mozilla\/5[.]0[ ][(]X11;[ ]Intel[ ]Mac[ ]OS[ ]X[ ]\d+[.]\d+;[ ]rv:\d{2,3}.0[)][ ]Gecko\/20100101[ ]Firefox\/\d{2,3}.0$/smx, + "\$firefox->agent(os => 'darwin', platform => 'X11') did return '$agent'"); + $firefox->agent(os => 'darwin', arch => '10.13'); + $agent = $firefox->agent(undef); + ok($agent =~ /^Mozilla\/5[.]0[ ][(]Macintosh;[ ]Intel[ ]Mac[ ]OS[ ]X[ ]10[.]13;[ ]rv:\d{2,3}.0[)][ ]Gecko\/20100101[ ]Firefox\/\d{2,3}.0$/smx, + "\$firefox->agent(os => 'darwin', arch => '10.13') did return '$agent'"); + $firefox->agent(os => 'freebsd', version => 118); + $agent = $firefox->agent(undef); + ok($agent eq $freebsd_118_user_agent_string, + "\$firefox->agent(os => 'freebsd', version => '118') did return '$agent'"); + eval { $firefox->agent(version => 'blah') }; + my $exception = $@; + chomp $exception; + ok(ref $@ eq 'Firefox::Marionette::Exception', "\$firefox->agent(version => 'blah') throws an exception:$exception"); + eval { $firefox->agent(increment => 'blah') }; + $exception = $@; + chomp $exception; + ok(ref $@ eq 'Firefox::Marionette::Exception', "\$firefox->agent(increment => 'blah') throws an exception:$exception"); + } + check_webdriver($firefox, $webdriver_definition_script, $webdriver_def_regex); + { + my $tmp_dir = File::Temp->newdir( + TEMPLATE => File::Spec->catdir(File::Spec->tmpdir(), 'perl_ff_m_test_XXXXXXXXXXX') + ) or die "Failed to create temporary directory:$!"; + local $ENV{HOME} = $tmp_dir->dirname();; + my $bcd_path = Firefox::Marionette::BCD_PATH(1); + ok($bcd_path, "Created $bcd_path for BCD file in $bcd_path"); + ok(1, "About to go to Firefox v122 with no BCD file available in $ENV{HOME}"); + ok($firefox->agent(version => 122), "\$firefox->agent(version => 122) with no BCD file available, but BCD_PATH(1) called"); + ok($firefox->agent(undef), "\$firefox->agent(undef) to reset agent string to original"); + } + { + my $tmp_dir = File::Temp->newdir( + TEMPLATE => File::Spec->catdir(File::Spec->tmpdir(), 'perl_ff_m_test_XXXXXXXXXXX') + ) or die "Failed to create temporary directory:$!"; + local $ENV{HOME} = $tmp_dir->dirname();; + ok(1, "About to go to Firefox v122 with no BCD file available in $ENV{HOME}"); + ok($firefox->agent(version => 122), "\$firefox->agent(version => 122) with no BCD file available and BCD_PATH(1) not called"); + ok($firefox->agent(undef), "\$firefox->agent(undef) to reset agent string to original"); + } + { + my %agent_parameters = ( + from => 'Mozilla/5.0 (X11; Linux x86_64; rv:20.0) Gecko/20100101 Firefox/125.0', + to => 'Mozilla/5.0 (X11; Linux x86_64; rv:20.0) Gecko/20100101 Firefox/100.0', + filters => qr/(?:ContentVisibilityAutoStateChangeEvent|withResolvers)/smxi + ); + my $javascript = Firefox::Marionette::Extension::Stealth->user_agent_contents(%agent_parameters); + ok($javascript =~ /delete[ ]window[.]ContentVisibilityAutoStateChangeEvent/, "Filtered extension code includes ContentVisibilityAutoStateChangeEvent"); + ok($javascript !~ /delete[ ]window[.]ShadowRoot/, "Filtered extension code does NOT include ShadowRoot"); + my $from = $agent_parameters{from}; + my $to = $agent_parameters{to}; + $agent_parameters{from} = $to; + $agent_parameters{to} = $from; + $javascript = Firefox::Marionette::Extension::Stealth->user_agent_contents(%agent_parameters); + ok($javascript =~ /Object.defineProperty[(]window[.]ContentVisibilityAutoStateChangeEvent/, "Filtered extension code includes ContentVisibilityAutoStateChangeEvent"); + ok($javascript !~ /Object.defineProperty[(]window[.]ShadowRoot/, "Filtered extension code does NOT include ShadowRoot"); + $agent_parameters{from} = $from; + $agent_parameters{to} = $to; + delete $agent_parameters{filters}; + $javascript = Firefox::Marionette::Extension::Stealth->user_agent_contents(%agent_parameters); + ok($javascript =~ /delete[ ]window[.]ContentVisibilityAutoStateChangeEvent/, "Extension code includes ContentVisibilityAutoStateChangeEvent"); + ok($javascript =~ /delete[ ]window[.]ShadowRoot/, "Extension code includes ShadowRoot"); +if (0) { + %agent_parameters = ( + from => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + to => 'Mozilla/5.0 (X11; Linux x86_64; rv:40.0) Gecko/20100101 Firefox/40.0', + ); + $javascript = Firefox::Marionette::Extension::Stealth->user_agent_contents(%agent_parameters); +} + } + foreach my $version (reverse (5 .. 124)) { + ok(1, "About to go to Firefox v$version"); + my $agent = $firefox->agent(version => $version); + ok($agent =~ /Firefox\/(\d+)/smx, "\$firefox->agent(version => $version) produces the actual agent string which contains Firefox version '$1'"); + check_webdriver($firefox, $webdriver_definition_script, $webdriver_def_regex); + ok($firefox->go("http://$nginx_listen:" . $nginx->port()), "Loaded browserfeatcl"); + $agent = $firefox->agent(); + ok($agent =~ /Firefox\/(\d+)/smx, "\$firefox->agent() contains Firefox version '$1' in the agent string (real version is " . $firefox->browser_version() . ")"); + my $test_result_version = $1; + my $extracted_report = $firefox->await(sub { $firefox->has_class('success') or $firefox->has_class('error') })->text(); + ok($extracted_report =~ /You[']re[ ]using[ ]Firefox[ ](\d+)(?:[.]\d+)?(?:[ ]\-[ ](\d+)(?:[.]9)?[!])?/smx, "browserfeatcl reports '$extracted_report' which matches Firefox"); + my ($min_version, $max_version) = ($1, $2); + if (defined $max_version) { + ok($min_version <= $version && $version <= $max_version, "browserfeatcl matches between $min_version and $max_version which includes fake version '$version'"); + } else { + ok($min_version == $version, "browserfeatcl matches $min_version which equals fake version '$version'"); + } + check_webdriver($firefox, $webdriver_definition_script, $webdriver_def_regex); + ok($firefox->agent(undef), "\$firefox->agent(undef) to reset agent string to original"); + } + foreach my $version (reverse (31 .. 121)) { + my $chrome_user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/$version.0.0.0 Safari/537.36"; + ok(1, "About to go to Chrome v$version - $chrome_user_agent"); + my $agent = $firefox->agent($chrome_user_agent); + ok($agent =~ /Firefox\/(\d+)/smx, "\$firefox->agent('$chrome_user_agent') produces the actual agent string which contains Firefox version '$1'"); + check_webdriver($firefox, $webdriver_definition_script, $webdriver_def_regex); + ok($firefox->go("http://$nginx_listen:" . $nginx->port()), "Loaded browserfeatcl"); + $agent = $firefox->agent(); + ok($agent =~ /Chrome\/$version/smx, "\$firefox->agent() contains Chrome version '$version' in the agent string (real version is Firefox v" . $firefox->browser_version() . ")"); + my $extracted_report = $firefox->await(sub { $firefox->has_class('success') or $firefox->has_class('error') })->text(); + ok($extracted_report =~ /You[']re[ ]using[ ]Chrom(?:e|ium)[ ](\d+)(?:[.]\d+)?(?:[ ]\-[ ](\d+)[!])?/smx, "browserfeatcl reports '$extracted_report' which matches Chrome"); + my ($min_version, $max_version) = ($1, $2); + if (defined $max_version) { + ok($min_version <= $version && $version <= $max_version, "browserfeatcl matches between $min_version and $max_version which includes fake version '$version'"); + } else { + ok($min_version == $version, "browserfeatcl matches $min_version which equals fake version '$version'"); + } + check_webdriver($firefox, $webdriver_definition_script, $webdriver_def_regex); + ok($firefox->agent(undef), "\$firefox->agent(undef) to reset agent string to original"); + } + foreach my $version (reverse (9 .. 17)) { + my $safari_user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_3) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/$version Safari/605.1.15"; + ok(1, "About to go to Safari v$version - $safari_user_agent"); + my $agent = $firefox->agent($safari_user_agent); + ok($agent =~ /Firefox\/(\d+)/smx, "\$firefox->agent('$safari_user_agent') produces the actual agent string which contains Firefox version '$1'"); + check_webdriver($firefox, $webdriver_definition_script, $webdriver_def_regex); + ok($firefox->go("http://$nginx_listen:" . $nginx->port()), "Loaded browserfeatcl"); + $agent = $firefox->agent(); + ok($agent =~ /Version\/$version[ ]Safari\/(\d+)/smx, "\$firefox->agent() contains Safari version '$version' in the agent string (real version is Firefox v" . $firefox->browser_version() . ")"); + my $extracted_report = $firefox->await(sub { $firefox->has_class('success') or $firefox->has_class('error') })->text(); + ok($extracted_report =~ /You[']re[ ]using[ ]Safari[ ](\d+)(?:[.]\d+)?(?:[ ]\-[ ](\d+)[.]9[!])?/smx, "browserfeatcl reports '$extracted_report' which matches Safari"); + my ($min_version, $max_version) = ($1, $2); + if (defined $max_version) { + ok($min_version <= $version && $version <= $max_version, "browserfeatcl matches between $min_version and $max_version which includes fake version '$version'"); + } else { + ok($min_version == $version, "browserfeatcl matches $min_version which equals fake version '$version'"); + } + check_webdriver($firefox, $webdriver_definition_script, $webdriver_def_regex); + ok($firefox->agent(undef), "\$firefox->agent(undef) to reset agent string to original"); + } + ok($firefox->quit() == 0, "\$firefox->quit() was successful()"); + ok($nginx->stop() == 0, "Stopped nginx on $nginx_listen:" . $nginx->port()); +} + +sub check_webdriver { + my ($firefox, $webdriver_definition_script, $webdriver_def_regex) = @_; + if ($firefox->script(q[if ('webdriver' in navigator) { return 1 } else { return 0 }])) { + ok($firefox->script('return navigator.webdriver') == JSON::false(), "\$firefox->script('return navigator.webdriver') returns false"); + my $stealth_webdriver_definition = $firefox->script($webdriver_definition_script); + my $quoted_webdriver_definition = $stealth_webdriver_definition; + $quoted_webdriver_definition =~ s/\n/\\n/smxg; + ok($stealth_webdriver_definition =~ /^$webdriver_def_regex$/smx, "Webdriver definition matches:$quoted_webdriver_definition"); + } else { + my $agent = $firefox->agent(); + ok(1, "Webdriver does not exist for " . $agent); + } + return; +} + +sub _check_navigator_attributes { + my ($firefox, $major_version, $user_agent, %user_agents_to_js) = @_; + my $count = 0; + KEY: foreach my $key (qw( + platform + appVersion + )) { + my $value = $firefox->script('return navigator.' . $key); + if ($user_agent =~ /^libwww[-]perl/smx) { + ok(defined $value, "navigator.$key is unchanged as '$value'"); + } elsif (defined $user_agents_to_js{$user_agent}{$key}) { + if (($value ne $user_agents_to_js{$user_agent}{$key}) && ($major_version < 62) && ($major_version > 59) && ($count <= 1)) { # firefox-60.0esr has blown up on this b/c of a seeming race condition + my $redo_seconds = 4; + $count += 1; + diag("The navigator.$key value is incorrect as '$value'. Waiting $redo_seconds seconds to try again"); + sleep $redo_seconds; + redo KEY; + } + ok($value eq $user_agents_to_js{$user_agent}{$key}, "navigator.$key is now '$user_agents_to_js{$user_agent}{$key}':$value"); + } else { + ok(!defined $value, "navigator.$key is undefined"); + } + } + if ($major_version > $min_stealth_version) { + $count = 0; + KEY2: foreach my $key (qw( + productSub + vendor + vendorSub + oscpu + )) { + my $value = $firefox->script('return navigator.' . $key); + if ($user_agent =~ /^libwww[-]perl/smx) { + ok(defined $value, "navigator.$key is unchanged as '$value'"); + } elsif (defined $user_agents_to_js{$user_agent}{$key}) { + if (($value ne $user_agents_to_js{$user_agent}{$key}) && ($major_version < 62) && ($major_version > 59) && ($count <= 1)) { # firefox-60.0esr has blown up on this b/c of a seeming race condition + my $redo_seconds = 4; + $count += 1; + diag("The navigator.$key value is incorrect as '$value'. Waiting $redo_seconds seconds to try again"); + sleep $redo_seconds; + redo KEY2; + } + ok($value eq $user_agents_to_js{$user_agent}{$key}, "navigator.$key is now '$user_agents_to_js{$user_agent}{$key}':$value"); + } else { + ok(!defined $value, "navigator.$key is undefined"); + } + } + } + my $value = $firefox->script('return navigator.userAgent'); + ok($user_agent eq $value, "navigator.userAgent is now '$user_agent':$value"); +} + +done_testing(); diff --git a/t/author/bulk_test.pl b/t/author/bulk_test.pl index 98e12e5..8a182e3 100755 --- a/t/author/bulk_test.pl +++ b/t/author/bulk_test.pl @@ -530,6 +530,7 @@ _multiple_attempts_execute($^X, [ ($devel_cover_inc ? $devel_cover_inc : ()), '-Ilib', '-wT', 't/04-proxy.t' ], {}); _multiple_attempts_execute($^X, [ ($devel_cover_inc ? $devel_cover_inc : ()), '-Ilib', 't/04-webauthn.t' ], {}); _multiple_attempts_execute($^X, [ ($devel_cover_inc ? $devel_cover_inc : ()), '-Ilib', 't/04-botd.t' ], {}); + _multiple_attempts_execute($^X, [ ($devel_cover_inc ? $devel_cover_inc : ()), '-Ilib', '-wT', 't/04-browserfeatcl.t' ], {}); while (_check_for_background_processes($background_pids, @servers)) { sleep 10; } diff --git a/t/manifest.t b/t/manifest.t index 31ff867..edfcba4 100644 --- a/t/manifest.t +++ b/t/manifest.t @@ -19,6 +19,7 @@ ok_manifest({filter => [qr/(?: fingerprintjs| servers[.]csv| BotD| + browserfeatcl| package\-lock[.]json| MYMETA[.]json[.]lock )/smx]}); diff --git a/t/stub.pl b/t/stub.pl index 6ba7949..451a6c8 100755 --- a/t/stub.pl +++ b/t/stub.pl @@ -49,6 +49,7 @@ my $capability_length = length $capabilities; syswrite $client, $capability_length . q[:] . $capabilities or die "Failed to write to socket:$!"; my $context = "content"; + my $addon_number = 0; while(1) { $request = _get_request($client); my $message_id = $request->[1]; @@ -57,7 +58,13 @@ _send_response_body($client, $response_body); last; } elsif ($request->[2] eq 'Addon:Install') { - syswrite $client, qq(79:[$response_type,$message_id,null,{"value":"6eea9fdc37a5d8fbcbbecd57ee7272669e828a31\@temporary-addon"}]) or die "Failed to write to socket:$!"; + $addon_number += 1; + my $response_body = qq([$response_type,$message_id,null,{"value":"6eea9fdc37a5d8fbcbbecd57ee7272669e828a3${addon_number}\@temporary-addon"}]); + _send_response_body($client, $response_body); + } elsif ($request->[2] eq 'Addon:Uninstall') { + $addon_number += 1; + my $response_body = qq([$response_type,$message_id,null,{"value":null}]); + _send_response_body($client, $response_body); } elsif ($request->[2] eq 'WebDriver:Print') { syswrite $client, qq(1475:[$response_type,$message_id,null,{"value":"JVBERi0xLjUKJbXtrvsKNCAwIG9iago8PCAvTGVuZ3RoIDUgMCBSCiAgIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlCj4+CnN0cmVhbQp4nDNUMABCXUMgYW5ppJCcy1XIFahQyGVkoWdsaqQApUxNTfWMDQwVzI0hdFGqQrhCHpehAggWpSvoJxoopBcT1pTGFcgFACsfF2cKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqCiAgIDc1CmVuZG9iagozIDAgb2JqCjw8CiAgIC9FeHRHU3RhdGUgPDwKICAgICAgL2EwIDw8IC9DQSAxIC9jYSAxID4+CiAgID4+Cj4+CmVuZG9iago2IDAgb2JqCjw8IC9UeXBlIC9PYmpTdG0KICAgL0xlbmd0aCA3IDAgUgogICAvTiAxCiAgIC9GaXJzdCA0CiAgIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlCj4+CnN0cmVhbQp4nD3NMQvCMBQE4L2/4hbnJlEUIXRoC8VBkOgmDiU+pEsSkkbsvzeJ1PG+d7wTYJWUqG+LI9SX8UXYgFdADp7MDA4GVeBMz2ls7Qf3RAx7LnA4CjzKsbNmTvWA3b8/eBsdpMwh599G0ZWuSf1ogstbeln5hNlHWlOXWj29J01qaDM2TfmvKNjoNQVsy2biL4KVMvQKZW5kc3RyZWFtCmVuZG9iago3IDAgb2JqCiAgIDE0NwplbmRvYmoKOCAwIG9iago8PCAvVHlwZSAvT2JqU3RtCiAgIC9MZW5ndGggMTEgMCBSCiAgIC9OIDMKICAgL0ZpcnN0IDE2CiAgIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlCj4+CnN0cmVhbQp4nE2PTQvCMBBE7/kVc7NFaHZrxQ+kF8WLCCLexEOosQZKt6QR1F+vRgSvs/OWNwxSM4xJMYGnrBYL6MOjs9A7U9teAdAbd+5xRA7CHqcYLeXWBrAqy0jsvJxvlfVIKuO8gDOeZAWSawhdP9c6prU33dVVfSa+TtPvG29NkDe2ladrGoO18/Yi97+rk3ZlgkWymueUj2jMIybiohgyDYjSn8JXemmCaaSOeBwA/lh/Si9o4j6UCmVuZHN0cmVhbQplbmRvYmoKMTEgMCBvYmoKICAgMTgxCmVuZG9iagoxMiAwIG9iago8PCAvVHlwZSAvWFJlZgogICAvTGVuZ3RoIDU3CiAgIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlCiAgIC9TaXplIDEzCiAgIC9XIFsxIDIgMl0KICAgL1Jvb3QgMTAgMCBSCiAgIC9JbmZvIDkgMCBSCj4+CnN0cmVhbQp4nBXKQQ0AIAwEwW1LCLyQgB1cIA8TeIPrZ7K5HPCe08CpYNxkJEdYEd6TmZemEG6xtMWGD8f2BIAKZW5kc3RyZWFtCmVuZG9iagpzdGFydHhyZWYKODYzCiUlRU9GCg=="}]) or die "Failed to write to socket:$!"; } elsif ($request->[2] eq 'WebDriver:TakeScreenshot') { @@ -69,8 +76,15 @@ my $response_body = qq([$response_type,$message_id,null,{"value":null}]); _send_response_body($client, $response_body); } elsif ($request->[2] eq 'WebDriver:ExecuteScript') { - my $now = time; - my $response_body = qq([$response_type,$message_id,null,{"value":{"guid":"root________","index":0,"type":2,"title":"","dateAdded":$now,"lastModified":$now,"childCount":5}}]); + my $response_body; + if ($request->[3]->{script} eq 'return navigator.userAgent') { + my $trimmed_browser_version = $browser_version; + $trimmed_browser_version =~ s/^(\d+(?:[.]\d+)?).*$/$1/smx; + $response_body = qq([$response_type,$message_id,null,{"value":"Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/$trimmed_browser_version"}]); + } else { + my $now = time; + $response_body = qq([$response_type,$message_id,null,{"value":{"guid":"root________","index":0,"type":2,"title":"","dateAdded":$now,"lastModified":$now,"childCount":5}}]); + } _send_response_body($client, $response_body); } else { die "Unsupported method in stub firefox"; diff --git a/t/syscall_tests.pm b/t/syscall_tests.pm index ae2a3c1..517c989 100644 --- a/t/syscall_tests.pm +++ b/t/syscall_tests.pm @@ -60,7 +60,11 @@ sub allow { sub run { my ($class, $expected_error_as_posix) = @_; my $cwd = Cwd::cwd(); - %parameters = ( binary => File::Spec->catfile($cwd, 't', 'stub.pl'), har => 1 ); + %parameters = ( + binary => File::Spec->catfile($cwd, 't', 'stub.pl'), + har => 1, + stealth => 1, + ); my $success = 0; while(!$success) { $syscall_count = 0; @@ -69,6 +73,7 @@ sub run { $firefox->pdf(); $firefox->selfie(); $firefox->import_bookmarks(File::Spec->catfile(Cwd::cwd(), qw(t data bookmarks_empty.html))); + $firefox->agent(version => 100); my $final = $syscall_error_at_count; $syscall_error_at_count = undef; ok($syscall_count >= 0 && $firefox->quit() == 0, "Firefox exited okay after $final successful $function calls"); diff --git a/t/test_daemons.pm b/t/test_daemons.pm index 920812a..40dec0f 100644 --- a/t/test_daemons.pm +++ b/t/test_daemons.pm @@ -550,6 +550,8 @@ sub new { my $username = $parameters{username}; my $password = $parameters{password}; my $realm = $parameters{realm}; + my $root_name = $parameters{htdocs}; + my $index_name = $parameters{index}; my $port = $class->new_port(); my $base_directory = $class->tmp_directory('nginx'); my $passwd_path = @@ -565,26 +567,36 @@ sub new { my $certificate_handle = $ca->new_cert( $key_path, $listen, $certificate_path ); } - my $root_name = 'htdocs'; - my $root_directory = - File::Spec->catfile( $base_directory->dirname(), $root_name ); - mkdir $root_directory, Fcntl::S_IRWXU() - or Carp::croak("Failed to mkdir $root_directory:$EXTENDED_OS_ERROR"); - my $index_name = 'index.txt'; - my $index_file_path = File::Spec->catfile( $root_directory, $index_name ); - my $index_handle = FileHandle->new( - $index_file_path, - Fcntl::O_WRONLY() | Fcntl::O_EXCL() | Fcntl::O_CREAT(), - Fcntl::S_IRUSR() | Fcntl::S_IWUSR() - ) or Carp::croak("Failed to open $index_file_path:$EXTENDED_OS_ERROR"); - my $random_string = - MIME::Base64::encode_base64( - Crypt::URandom::urandom( _RANDOM_STRING_LENGTH() ) ); - chomp $random_string; - print {$index_handle} $random_string - or Carp::croak("Failed to write to $index_file_path:$EXTENDED_OS_ERROR"); - close $index_handle - or Carp::croak("Failed to close $index_file_path:$EXTENDED_OS_ERROR"); + my $random_string; + if ( !$root_name ) { + $root_name = 'htdocs'; + my $root_directory = + File::Spec->catfile( $base_directory->dirname(), $root_name ); + mkdir $root_directory, Fcntl::S_IRWXU() + or Carp::croak("Failed to mkdir $root_directory:$EXTENDED_OS_ERROR"); + if ( !$index_name ) { + $index_name = 'index.txt'; + my $index_file_path = + File::Spec->catfile( $root_directory, $index_name ); + my $index_handle = FileHandle->new( + $index_file_path, + Fcntl::O_WRONLY() | Fcntl::O_EXCL() | Fcntl::O_CREAT(), + Fcntl::S_IRUSR() | Fcntl::S_IWUSR() + ) + or + Carp::croak("Failed to open $index_file_path:$EXTENDED_OS_ERROR"); + $random_string = + MIME::Base64::encode_base64( + Crypt::URandom::urandom( _RANDOM_STRING_LENGTH() ) ); + chomp $random_string; + print {$index_handle} $random_string + or Carp::croak( + "Failed to write to $index_file_path:$EXTENDED_OS_ERROR"); + close $index_handle + or Carp::croak( + "Failed to close $index_file_path:$EXTENDED_OS_ERROR"); + } + } my $pid_path = File::Spec->catfile( $base_directory->dirname(), 'nginx.pid' ); my $pid_handle = FileHandle->new( @@ -640,6 +652,12 @@ http { keepalive_timeout 65; types_hash_max_size 4096; + types { + text/html html; + text/javascript js; + text/css css; + application/json json; + } default_type text/plain; server {