Skip to content

Commit a7c947c

Browse files
haarghaarg
authored
Merge pull request #1299 from metacpan/haarg/es5-compat
various fixes for Elasticsearch 5 compatibility
2 parents 859022a + b621a68 commit a7c947c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+583
-417
lines changed

‎cpanfile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ requires 'Safe', '2.35'; # bug fixes (used by Parse::PMFile)
124124
requires 'Scalar::Util', '1.62'; # Moose
125125
requires 'Search::Elasticsearch' => '8.12';
126126
requires 'Search::Elasticsearch::Client::2_0' => '6.81';
127+
requires 'Search::Elasticsearch::Client::5_0' => '6.81';
127128
requires 'Term::Choose', '1.754'; # Git::Helpers
128129
requires 'Throwable::Error';
129130
requires 'Term::Size::Any'; # for Catalyst

‎cpanfile.snapshot

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6480,6 +6480,38 @@ DISTRIBUTIONS
64806480
namespace::clean 0
64816481
strict 0
64826482
warnings 0
6483+
Search-Elasticsearch-Client-5_0-6.81
6484+
pathname: E/EZ/EZIMUEL/Search-Elasticsearch-Client-5_0-6.81.tar.gz
6485+
provides:
6486+
Search::Elasticsearch::Client::5_0 6.81
6487+
Search::Elasticsearch::Client::5_0::Bulk 6.81
6488+
Search::Elasticsearch::Client::5_0::Direct 6.81
6489+
Search::Elasticsearch::Client::5_0::Direct::Cat 6.81
6490+
Search::Elasticsearch::Client::5_0::Direct::Cluster 6.81
6491+
Search::Elasticsearch::Client::5_0::Direct::Indices 6.81
6492+
Search::Elasticsearch::Client::5_0::Direct::Ingest 6.81
6493+
Search::Elasticsearch::Client::5_0::Direct::Nodes 6.81
6494+
Search::Elasticsearch::Client::5_0::Direct::Snapshot 6.81
6495+
Search::Elasticsearch::Client::5_0::Direct::Tasks 6.81
6496+
Search::Elasticsearch::Client::5_0::Role::API 6.81
6497+
Search::Elasticsearch::Client::5_0::Role::Bulk 6.81
6498+
Search::Elasticsearch::Client::5_0::Role::Scroll 6.81
6499+
Search::Elasticsearch::Client::5_0::Scroll 6.81
6500+
Search::Elasticsearch::Client::5_0::TestServer 6.81
6501+
requirements:
6502+
Devel::GlobalDestruction 0
6503+
ExtUtils::MakeMaker 0
6504+
Moo 0
6505+
Moo::Role 0
6506+
Search::Elasticsearch 6.00
6507+
Search::Elasticsearch::Role::API 0
6508+
Search::Elasticsearch::Role::Client::Direct 0
6509+
Search::Elasticsearch::Role::Is_Sync 0
6510+
Search::Elasticsearch::Util 0
6511+
Try::Tiny 0
6512+
namespace::clean 0
6513+
strict 0
6514+
warnings 0
64836515
Sereal-Decoder-5.004
64846516
pathname: Y/YV/YVES/Sereal-Decoder-5.004.tar.gz
64856517
provides:

‎lib/Catalyst/Plugin/Session/Store/ElasticSearch.pm

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ extends 'Catalyst::Plugin::Session::Store';
77
use MooseX::Types::ElasticSearch qw( ES );
88

99
use MetaCPAN::Server::Config ();
10+
use MetaCPAN::Util qw( true false );
1011

1112
has _session_es => (
1213
is => 'ro',
@@ -55,7 +56,7 @@ sub store_session_data {
5556
type => $self->_session_es_type,
5657
id => $sid,
5758
body => $session,
58-
refresh => 1,
59+
refresh => true,
5960
);
6061
}
6162
}
@@ -68,7 +69,7 @@ sub delete_session_data {
6869
index => $self->_session_es_index,
6970
type => $self->_session_es_type,
7071
id => $sid,
71-
refresh => 1,
72+
refresh => true,
7273
);
7374
};
7475
}

‎lib/ElasticSearchX/Model/Document/Set.pm

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
package ElasticSearchX::Model::Document::Set;
2+
use strict;
3+
use warnings;
4+
5+
use MetaCPAN::Model::Hacks;
6+
7+
no warnings 'redefine';
8+
9+
our %query_override;
10+
my $_build_query = \&_build_query;
11+
*_build_query = sub {
12+
my $query = $_build_query->(@_);
13+
%$query = ( %$query, %query_override, );
14+
return $query;
15+
};
16+
17+
our %qs_override;
18+
my $_build_qs = \&_build_qs;
19+
*_build_qs = sub {
20+
my $qs = $_build_qs->(@_);
21+
%$qs = ( %$qs, %qs_override, );
22+
return $qs;
23+
};
24+
25+
my $delete = \&delete;
26+
*delete = sub {
27+
local %qs_override = ( search_type => 'query_then_fetch' );
28+
local %query_override = ( sort => '_doc' );
29+
return $delete->(@_);
30+
};
31+
32+
my $get = \&get;
33+
*get = sub {
34+
my ( $self, $args, $qs ) = @_;
35+
if ( $self->es->api_version eq '2_0' ) {
36+
goto &$get;
37+
}
38+
my %qs = %{ $qs || {} };
39+
if ( my $fields = $self->fields ) {
40+
$qs{_source} = $fields;
41+
local $self->{fields};
42+
return $get->( $self, $args, \%qs );
43+
}
44+
goto &$get;
45+
};
46+
47+
1;

‎lib/MetaCPAN/API/Model/Cover.pm

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package MetaCPAN::API::Model::Cover;
22

33
use MetaCPAN::Moose;
44

5+
use MetaCPAN::Util qw(hit_total);
6+
57
with 'MetaCPAN::API::Model::Role::ES';
68

79
sub find_release_coverage {
@@ -17,7 +19,7 @@ sub find_release_coverage {
1719
size => 999,
1820
}
1921
);
20-
$res->{hits}{total} or return {};
22+
hit_total($res) or return {};
2123

2224
return +{
2325
%{ $res->{hits}{hits}[0]{_source} },

‎lib/MetaCPAN/Document/File.pm

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -414,9 +414,8 @@ sub _build_suggest {
414414
$weight = 0 if $weight < 0;
415415

416416
return +{
417-
input => [$doc],
418-
payload => { doc_name => $doc },
419-
weight => $weight,
417+
input => [$doc],
418+
weight => $weight,
420419
};
421420
}
422421

‎lib/MetaCPAN/Document/File/Set.pm

Lines changed: 61 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use List::Util qw( max );
66
use MetaCPAN::Query::Favorite ();
77
use MetaCPAN::Query::File ();
88
use MetaCPAN::Query::Release ();
9-
use MetaCPAN::Util qw( single_valued_arrayref_to_scalar true false );
9+
use MetaCPAN::Util qw( true false );
1010

1111
extends 'ElasticSearchX::Model::Document::Set';
1212

@@ -75,7 +75,8 @@ my @ROGUE_DISTRIBUTIONS = qw(
7575

7676
sub find {
7777
my ( $self, $module ) = @_;
78-
my @candidates = $self->index->type('file')->query( {
78+
79+
my $query = {
7980
bool => {
8081
must => [
8182
{ term => { indexed => true } },
@@ -119,22 +120,38 @@ sub find {
119120
},
120121
],
121122
},
122-
} )->sort( [
123-
'_score',
124-
{ 'version_numified' => { order => 'desc' } },
125-
{ 'date' => { order => 'desc' } },
126-
{ 'mime' => { order => 'asc' } },
127-
{ 'stat.mtime' => { order => 'desc' } }
128-
] )->search_type('dfs_query_then_fetch')->size(100)->all;
123+
};
124+
125+
my $res = $self->es->search(
126+
index => $self->index->name,
127+
type => 'file',
128+
search_type => 'dfs_query_then_fetch',
129+
body => {
130+
query => $query,
131+
sort => [
132+
'_score',
133+
{ 'version_numified' => { order => 'desc' } },
134+
{ 'date' => { order => 'desc' } },
135+
{ 'mime' => { order => 'asc' } },
136+
{ 'stat.mtime' => { order => 'desc' } }
137+
],
138+
_source => [
139+
qw( documentation module.indexed module.authoried module.name )
140+
],
141+
size => 100,
142+
},
143+
);
144+
145+
my @candidates = @{ $res->{hits}{hits} };
129146

130147
my ($file) = grep {
131-
grep { $_->indexed && $_->authorized && $_->name eq $module }
132-
@{ $_->module || [] }
133-
} grep { !$_->documentation || $_->documentation eq $module }
148+
grep { $_->{indexed} && $_->{authorized} && $_->{name} eq $module }
149+
@{ $_->{module} || [] }
150+
} grep { !$_->{documentation} || $_->{documentation} eq $module }
134151
@candidates;
135152

136153
$file ||= shift @candidates;
137-
return $file ? $self->get( $file->id ) : undef;
154+
return $file ? $self->get( $file->{_id} ) : undef;
138155
}
139156

140157
sub find_pod {
@@ -265,15 +282,13 @@ sub history {
265282

266283
sub autocomplete {
267284
my ( $self, @terms ) = @_;
268-
my $query = join( q{ }, @terms );
269-
return $self unless $query;
270285

271-
my $data = $self->search_type('dfs_query_then_fetch')->query( {
286+
my $query = {
272287
bool => {
273288
must => [
274289
{
275290
multi_match => {
276-
query => $query,
291+
query => join( q{ }, @terms ),
277292
type => 'most_fields',
278293
fields => [ 'documentation', 'documentation.*' ],
279294
analyzer => 'camelcase',
@@ -288,15 +303,21 @@ sub autocomplete {
288303
must_not =>
289304
[ { terms => { distribution => \@ROGUE_DISTRIBUTIONS } }, ],
290305
},
291-
} )->sort( [ '_score', 'documentation' ] );
292-
293-
$data = $data->fields( [qw(documentation release author distribution)] )
294-
unless $self->fields;
295-
296-
$data = $data->source(0)->raw->all;
306+
};
307+
308+
my $data = $self->es->search(
309+
search_type => 'dfs_query_then_fetch',
310+
index => $self->index->name,
311+
type => 'file',
312+
body => {
313+
query => $query,
314+
sort => [ '_score', 'documentation' ],
315+
_source => [qw( documentation release author distribution )],
316+
},
317+
);
297318

298-
single_valued_arrayref_to_scalar( $_->{fields} )
299-
for @{ $data->{hits}{hits} };
319+
# this is backcompat. we don't use this end point.
320+
$_->{fields} = delete $_->{_source} for @{ $data->{hits}{hits} };
300321

301322
return $data;
302323
}
@@ -307,8 +328,7 @@ sub autocomplete_suggester {
307328

308329
my $search_size = 100;
309330

310-
my $suggestions
311-
= $self->search_type('dfs_query_then_fetch')->es->suggest( {
331+
my $suggestions = $self->es->suggest( {
312332
index => $self->index->name,
313333
body => {
314334
documentation => {
@@ -319,7 +339,7 @@ sub autocomplete_suggester {
319339
}
320340
}
321341
},
322-
} );
342+
} );
323343

324344
my %docs;
325345

@@ -328,14 +348,6 @@ sub autocomplete_suggester {
328348
( $docs{ $suggest->{text} }, $suggest->{score} );
329349
}
330350

331-
my @fields = qw(
332-
author
333-
date
334-
deprecated
335-
distribution
336-
documentation
337-
release
338-
);
339351
my $data = $self->es->search( {
340352
index => $self->index->name,
341353
type => 'file',
@@ -355,17 +367,22 @@ sub autocomplete_suggester {
355367
],
356368
}
357369
},
370+
_source => [ qw(
371+
author
372+
date
373+
deprecated
374+
distribution
375+
documentation
376+
release
377+
) ],
378+
size => $search_size,
358379
},
359-
fields => \@fields,
360-
size => $search_size,
361380
} );
362381

363382
my %valid = map {
364-
my $got = $_->{fields};
365-
my %record;
366-
@record{@fields} = map { $got->{$_}[0] } @fields;
367-
$record{name} = delete $record{documentation}; # rename
368-
( $_->{fields}{documentation}[0] => \%record );
383+
my %record = %{ $_->{_source} };
384+
$record{name} = delete $record{documentation}; # rename
385+
( $record{name} => \%record );
369386
} @{ $data->{hits}{hits} };
370387

371388
# remove any exact match, it will be added later
@@ -393,7 +410,7 @@ sub autocomplete_suggester {
393410
sub find_changes_files {
394411
my ( $self, $author, $release ) = @_;
395412
my $result = $self->files_by_category( $author, $release, ['changelog'],
396-
{ fields => true } );
413+
{ _source => true } );
397414
my ($file) = @{ $result->{categories}{changelog} || [] };
398415
return $file;
399416
}

‎lib/MetaCPAN/Document/Mirror.pm

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use strict;
44
use warnings;
55

66
use Moose;
7-
use ElasticSearchX::Model::Document::Types qw( Location );
7+
use MooseX::Types::ElasticSearch qw( Location );
88
use ElasticSearchX::Model::Document;
99

1010
use MetaCPAN::Types::TypeTiny qw( Dict Str );

‎lib/MetaCPAN/Document/Release/Set.pm

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ package MetaCPAN::Document::Release::Set;
22

33
use Moose;
44

5-
use MetaCPAN::Util qw( single_valued_arrayref_to_scalar );
6-
75
use MetaCPAN::Query::Release ();
86

97
extends 'ElasticSearchX::Model::Document::Set';

‎lib/MetaCPAN/Model/Hacks.pm

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package MetaCPAN::Model::Hacks;
2+
use strict;
3+
use warnings;
4+
5+
sub import {
6+
my ( $caller, $caller_file ) = caller;
7+
8+
my $file = $caller =~ s{::}{/}gr . '.pm';
9+
my $dir = $caller_file =~ s{/\Q$file\E\z}{}r;
10+
local @INC = grep $_ ne $dir, @INC;
11+
my $inc;
12+
{
13+
local $INC{$file};
14+
delete $INC{$file};
15+
require $file;
16+
$inc = $INC{$file};
17+
}
18+
delete $INC{$file};
19+
$INC{$file} = $inc;
20+
return;
21+
}
22+
23+
1;

‎lib/MetaCPAN/Model/Release.pm

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,8 @@ sub _build_document {
222222
|| $document->{abstract} eq 'null' );
223223

224224
$document
225-
= $self->index->type('release')->put( $document, { refresh => 1 } );
225+
= $self->index->type('release')
226+
->put( $document, { refresh => true } );
226227

227228
# create distribution if doesn't exist
228229
my $dist_count = $self->es->count(

0 commit comments

Comments
 (0)