Skip to content

Commit

Permalink
Functional fetch support
Browse files Browse the repository at this point in the history
  • Loading branch information
@adamziel
adamziel committed Jan 6, 2025
1 parent bb3928e commit b28b46a
Show file tree
Hide file tree
Showing 6 changed files with 417 additions and 95 deletions.
3 changes: 3 additions & 0 deletions packages/playground/data-liberation/bootstrap.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
require_once __DIR__ . '/blueprints-library/src/WordPress/AsyncHttp/HttpError.php';
require_once __DIR__ . '/blueprints-library/src/WordPress/AsyncHttp/Connection.php';
require_once __DIR__ . '/blueprints-library/src/WordPress/AsyncHttp/Client.php';
require_once __DIR__ . '/blueprints-library/src/WordPress/AsyncHttp/ResponseWriter/ResponseWriter.php';
require_once __DIR__ . '/blueprints-library/src/WordPress/AsyncHttp/ResponseWriter/StreamingResponseWriter.php';
require_once __DIR__ . '/blueprints-library/src/WordPress/AsyncHttp/ResponseWriter/BufferingResponseWriter.php';

require_once __DIR__ . '/blueprints-library/src/WordPress/Filesystem/WP_Abstract_Filesystem.php';
require_once __DIR__ . '/blueprints-library/src/WordPress/Filesystem/WP_Local_Filesystem.php';
Expand Down
8 changes: 4 additions & 4 deletions packages/playground/data-liberation/src/git/WP_Git_Client.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ public function list_objects($ref_hash) {
'Content-Type: application/x-git-upload-pack-request',
]);

$pack_data = $this->accumulate_pack_data_from_multiplexed_chunks($response);
$pack_data = self::accumulate_pack_data_from_multiplexed_chunks($response);
return WP_Git_Pack_Processor::decode($pack_data);
}

Expand Down Expand Up @@ -208,7 +208,7 @@ public function fetchObjects($refs) {
'Accept: application/x-git-upload-pack-advertisement',
'Content-Type: application/x-git-upload-pack-request',
]);
$pack_data = $this->accumulate_pack_data_from_multiplexed_chunks($response);
$pack_data = self::accumulate_pack_data_from_multiplexed_chunks($response);
WP_Git_Pack_Processor::decode($pack_data, $this->index);
return true;
}
Expand Down Expand Up @@ -256,9 +256,9 @@ private function encode_packet_line($data) {
return str_pad(dechex($length), 4, '0', STR_PAD_LEFT) . $data;
}

private function accumulate_pack_data_from_multiplexed_chunks($raw_response) {
static public function accumulate_pack_data_from_multiplexed_chunks($raw_response) {
$parsed_pack_data = [];
$parsed_chunks = $this->parse_multiplexed_pack_data($raw_response);
$parsed_chunks = self::parse_multiplexed_pack_data($raw_response);
foreach($parsed_chunks as $chunk) {
if($chunk['type'] !== 'side-band') {
continue;
Expand Down
104 changes: 51 additions & 53 deletions packages/playground/data-liberation/src/git/WP_Git_Pack_Processor.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,16 +155,15 @@ static private function wrap_object($type, $object) {
static public function encode_packet_lines(array $payloads): string {
$lines = [];
foreach($payloads as $payload) {
if($payload === '0000' || $payload === '0001' || $payload === '0002') {
$lines[] = $payload;
} else {
$lines[] = self::encode_packet_line($payload);
}
$lines[] = self::encode_packet_line($payload);
}
return implode('', $lines);
}

static public function encode_packet_line(string $payload): string {
if($payload === '0000' || $payload === '0001' || $payload === '0002') {
return $payload;
}
$length = strlen($payload) + 4;
return sprintf("%04x", $length) . $payload;
}
Expand All @@ -179,55 +178,9 @@ static public function decode($pack_bytes, $pack_index=null) {
}

$parsed_pack = self::parse_pack_data($pack_bytes);
$objects = $parsed_pack['objects'];

$by_oid = [];
$by_offset = [];
$resolved_objects = 0;
// Index entities and resolve deltas
// Run until all objects are resolved
while($resolved_objects < count($objects)) {
$resolved_in_this_iteration = 0;
for($i = 0; $i < count($objects); $i++) {
// Skip already processed objects
if(
isset($by_offset[$objects[$i]['header_offset']]) &&
isset($by_oid[$objects[$i]['oid']])
) {
continue;
}

if($objects[$i]['type'] === self::OBJECT_TYPE_OFS_DELTA) {
$target_offset = $objects[$i]['header_offset'] - $objects[$i]['ofs'];
if(!isset($by_offset[$target_offset])) {
continue;
}
// TODO: Make sure the base object will never be another delta.
$base = $objects[$by_offset[$target_offset]];
$objects[$i]['content'] = self::applyDelta($base['content'], $objects[$i]['content']);
$objects[$i]['type'] = $base['type'];
} else if($objects[$i]['type'] === self::OBJECT_TYPE_REF_DELTA) {
if(!isset($by_oid[$objects[$i]['reference']])) {
continue;
}
$base = $objects[$by_oid[$objects[$i]['reference']]];
$objects[$i]['content'] = self::applyDelta($base['content'], $objects[$i]['content']);
$objects[$i]['type'] = $base['type'];
}
$oid = sha1(self::wrap_git_object($objects[$i]['type'], $objects[$i]['content']));
$objects[$i]['oid'] = $oid;
$by_oid[$oid] = $i;
$by_offset[$objects[$i]['header_offset']] = $i;
++$resolved_in_this_iteration;
++$resolved_objects;
}
if($resolved_in_this_iteration === 0) {
throw new Exception('Could not resolve objects');
}
}

// Resolve trees
foreach($objects as $object) {
foreach($parsed_pack['objects'] as $object) {
$pack_index->add_object($object['type'], $object['content']);
}

Expand Down Expand Up @@ -402,7 +355,7 @@ static private function readVariableLength($data, &$offset) {
return $result;
}

static private function parse_pack_data($packData) {
static public function parse_pack_data($packData) {
$offset = 0;

// Basic sanity checks
Expand Down Expand Up @@ -435,6 +388,51 @@ static private function parse_pack_data($packData) {
$object['content'] = self::inflate_object($packData, $offset, $object['uncompressed_length']);
$objects[] = $object;
}

$by_oid = [];
$by_offset = [];
$resolved_objects = 0;
// Index entities and resolve deltas
// Run until all objects are resolved
while($resolved_objects < count($objects)) {
$resolved_in_this_iteration = 0;
for($i = 0; $i < count($objects); $i++) {
// Skip already processed objects
if(
isset($by_offset[$objects[$i]['header_offset']]) &&
isset($by_oid[$objects[$i]['oid']])
) {
continue;
}

if($objects[$i]['type'] === self::OBJECT_TYPE_OFS_DELTA) {
$target_offset = $objects[$i]['header_offset'] - $objects[$i]['ofs'];
if(!isset($by_offset[$target_offset])) {
continue;
}
// TODO: Make sure the base object will never be another delta.
$base = $objects[$by_offset[$target_offset]];
$objects[$i]['content'] = self::applyDelta($base['content'], $objects[$i]['content']);
$objects[$i]['type'] = $base['type'];
} else if($objects[$i]['type'] === self::OBJECT_TYPE_REF_DELTA) {
if(!isset($by_oid[$objects[$i]['reference']])) {
continue;
}
$base = $objects[$by_oid[$objects[$i]['reference']]];
$objects[$i]['content'] = self::applyDelta($base['content'], $objects[$i]['content']);
$objects[$i]['type'] = $base['type'];
}
$oid = sha1(self::wrap_git_object($objects[$i]['type'], $objects[$i]['content']));
$objects[$i]['oid'] = $oid;
$by_oid[$oid] = $i;
$by_offset[$objects[$i]['header_offset']] = $i;
++$resolved_in_this_iteration;
++$resolved_objects;
}
if($resolved_in_this_iteration === 0) {
throw new Exception('Could not resolve objects');
}
}
return [
'objects' => $objects,
'total_objects' => $objectCount,
Expand Down
17 changes: 12 additions & 5 deletions packages/playground/data-liberation/src/git/WP_Git_Repository.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ class WP_Git_Repository {
private $diff_engine;

private const DELETE_PLACEHOLDER = 'DELETE_PLACEHOLDER';
private const NULL_OID = '0000000000000000000000000000000000000000';
public const NULL_OID = '0000000000000000000000000000000000000000';

public function __construct(
WP_Abstract_Filesystem $fs,
Expand Down Expand Up @@ -372,7 +372,7 @@ public function oid_exists($oid) {
public function read_by_path($path, $root_tree_oid=null) {
if($root_tree_oid === null) {
$head_oid = $this->get_ref_head('HEAD');
if(false === $this->read_object($head_oid)) {
if(!$head_oid || false === $this->read_object($head_oid)) {
return false;
}
$root_tree_oid = $this->get_parsed_commit()['tree'] ?? null;
Expand Down Expand Up @@ -433,7 +433,7 @@ public function find_path_descendants($path) {
return $oids;
}

public function find_objects_added_in($new_tree_oid, $old_tree_oid=null, $options=[]) {
public function find_objects_added_in($new_tree_oid, $old_tree_oid=WP_Git_Repository::NULL_OID, $options=[]) {
$old_tree_index = $options['old_tree_index'] ?? $this;
if($old_tree_index === null) {
$old_tree_index = $this;
Expand All @@ -452,7 +452,7 @@ public function find_objects_added_in($new_tree_oid, $old_tree_oid=null, $option
}

// Resolve the actual tree oid if $old_tree_oid is a commit
if($old_tree_oid) {
if(!$this->is_null_oid($old_tree_oid)) {
if(false === $old_tree_index->read_object($old_tree_oid)) {
$this->last_error = 'Failed to read object: ' . $old_tree_oid;
return false;
Expand All @@ -475,6 +475,9 @@ public function find_objects_added_in($new_tree_oid, $old_tree_oid=null, $option
if($current_new_oid === $current_old_oid) {
continue;
}
if($this->is_null_oid($current_new_oid)) {
continue;
}

if(false === $this->read_object($current_new_oid)) {
$this->last_error = 'Failed to read object: ' . $current_new_oid;
Expand All @@ -492,7 +495,7 @@ public function find_objects_added_in($new_tree_oid, $old_tree_oid=null, $option
yield $this->get_oid();

$old_tree = [];
if($current_old_oid) {
if(!$this->is_null_oid($current_old_oid)) {
if(false === $old_tree_index->read_object($current_old_oid)) {
$this->last_error = 'Failed to read object: ' . $current_old_oid;
return false;
Expand All @@ -506,6 +509,10 @@ public function find_objects_added_in($new_tree_oid, $old_tree_oid=null, $option
}
}

private function is_null_oid($oid) {
return $oid === null || $oid === WP_Git_Repository::NULL_OID;
}

public function set_ref_head($ref, $oid) {
$path = $this->resolve_ref_file_path($ref);
if(!$path) {
Expand Down
Loading

0 comments on commit b28b46a

Please sign in to comment.