xref: /aosp_15_r20/external/cldr/tools/cldr-code/src/main/java/org/unicode/cldr/util/VoteResolver.java (revision 912701f9769bb47905792267661f0baf2b85bed5)
1 package org.unicode.cldr.util;
2 
3 import com.google.common.base.Objects;
4 import com.google.common.collect.ImmutableSet;
5 import com.ibm.icu.text.Collator;
6 import com.ibm.icu.util.Output;
7 import com.ibm.icu.util.ULocale;
8 import java.sql.Timestamp;
9 import java.util.*;
10 import java.util.Map.Entry;
11 import java.util.regex.Matcher;
12 import java.util.regex.Pattern;
13 import org.unicode.cldr.icu.LDMLConstants;
14 import org.unicode.cldr.test.CheckCLDR;
15 import org.unicode.cldr.test.CheckCLDR.Phase;
16 import org.unicode.cldr.test.CheckWidths;
17 import org.unicode.cldr.test.DisplayAndInputProcessor;
18 
19 /**
20  * This class implements the vote resolution process agreed to by the CLDR committee. Here is an
21  * example of usage:
22  *
23  * <pre>
24  * // before doing anything, initialize the voter data (who are the voters at what levels) with setVoterToInfo.
25  * // We assume this doesn't change often
26  * // here is some fake data:
27  * VoteResolver.setVoterToInfo(Utility.asMap(new Object[][] {
28  *     { 666, new VoterInfo(Organization.google, Level.vetter, &quot;J. Smith&quot;) },
29  *     { 555, new VoterInfo(Organization.google, Level.street, &quot;S. Jones&quot;) },
30  *     { 444, new VoterInfo(Organization.google, Level.vetter, &quot;S. Samuels&quot;) },
31  *     { 333, new VoterInfo(Organization.apple, Level.vetter, &quot;A. Mutton&quot;) },
32  *     { 222, new VoterInfo(Organization.adobe, Level.expert, &quot;A. Aldus&quot;) },
33  *     { 111, new VoterInfo(Organization.ibm, Level.street, &quot;J. Henry&quot;) }, }));
34  *
35  * // you can create a resolver and keep it around. It isn't thread-safe, so either have a separate one per thread (they
36  * // are small), or synchronize.
37  * VoteResolver resolver = new VoteResolver();
38  *
39  * // For any particular base path, set the values
40  * // set the 1.5 status (if we're working on 1.6). This &lt;b&gt;must&lt;/b&gt; be done for each new base path
41  * resolver.newPath(oldValue, oldStatus);
42  * [TODO: function newPath doesn't exist, revise this documentation]
43  *
44  * // now add some values, with who voted for them
45  * resolver.add(value1, voter1);
46  * resolver.add(value1, voter2);
47  * resolver.add(value2, voter3);
48  *
49  * // Once you've done that, you can get the results for the base path
50  * winner = resolver.getWinningValue();
51  * status = resolver.getWinningStatus();
52  * conflicts = resolver.getConflictedOrganizations();
53  * </pre>
54  */
55 public class VoteResolver<T> {
56     public static final boolean DROP_HARD_INHERITANCE = true;
57 
58     private final VoterInfoList voterInfoList;
59 
VoteResolver(VoterInfoList vil)60     public VoteResolver(VoterInfoList vil) {
61         voterInfoList = vil;
62     }
63 
64     private static final boolean DEBUG = false;
65 
66     /** This enables a prose discussion of the voting process. */
67     private DeferredTranscript transcript = null;
68 
enableTranscript()69     public void enableTranscript() {
70         if (transcript == null) {
71             transcript = new DeferredTranscript();
72         }
73     }
74 
disableTranscript()75     public void disableTranscript() {
76         transcript = null;
77     }
78 
getTranscript()79     public String getTranscript() {
80         if (transcript == null) {
81             return null;
82         } else {
83             return transcript.get();
84         }
85     }
86 
87     /**
88      * Add an annotation
89      *
90      * @param fmt
91      * @param args
92      */
annotateTranscript(String fmt, Object... args)93     private final void annotateTranscript(String fmt, Object... args) {
94         if (transcript != null) {
95             transcript.add(fmt, args);
96         }
97     }
98 
99     /**
100      * A placeholder for winningValue when it would otherwise be null. It must match
101      * NO_WINNING_VALUE in the client JavaScript code.
102      */
103     private static final String NO_WINNING_VALUE = "no-winning-value";
104 
105     /**
106      * The status levels according to the committee, in ascending order
107      *
108      * <p>Status corresponds to icons as follows: A checkmark means it’s approved and is slated to
109      * be used. A cross means it’s a missing value. Green/orange check: The item has enough votes to
110      * be used in CLDR. Red/orange/black X: The item does not have enough votes to be used in CLDR,
111      * by most implementations (or is completely missing). Reference: <a
112      * href="https://cldr.unicode.org/translation/getting-started/guide">guide</a>
113      *
114      * <p>When the item is inherited, i.e., winningValue is INHERITANCE_MARKER (↑↑↑), then
115      * orange/red X are replaced by orange/red up-arrow. That change is made only on the client.
116      *
117      * <p>Status.approved: green check Status.contributed: orange check Status.provisional: orange X
118      * (or orange up-arrow if inherited) Status.unconfirmed: red X (or red up-arrow if inherited
119      * Status.missing: black X
120      *
121      * <p>Not to be confused with VoteResolver.VoteStatus
122      */
123     public enum Status {
124         missing,
125         unconfirmed,
126         provisional,
127         contributed,
128         approved;
129 
fromString(String source)130         public static Status fromString(String source) {
131             return source == null ? missing : Status.valueOf(source);
132         }
133     }
134 
135     /**
136      * This is the "high bar" level where flagging is required.
137      *
138      * @see #getRequiredVotes()
139      */
140     public static final int HIGH_BAR = Level.tc.votes;
141 
142     public static final int LOWER_BAR = (2 * Level.vetter.votes);
143 
144     /**
145      * This is the level at which a vote counts. Each level also contains the weight.
146      *
147      * <p>Code related to Level.expert removed 2021-05-18 per CLDR-14597
148      */
149     public enum Level {
150         locked(0 /* votes */, 999 /* stlevel */),
151         guest(1 /* votes */, 10 /* stlevel */),
152         anonymous(0 /* votes */, 8 /* stlevel */),
153         vetter(4 /* votes */, 5 /* stlevel */, /* tcorgvotes */ 6), // org dependent- see getVotes()
154         // Manager and below can manage users
155         manager(4 /* votes */, 2 /* stlevel */),
156         tc(50 /* votes */, 1 /* stlevel */),
157         admin(100 /* votes */, 0 /* stlevel */);
158 
159         /**
160          * PERMANENT_VOTES is used by TC voters to "lock" locale+path permanently (including future
161          * versions, until unlocked), in the current VOTE_VALUE table. It is public for
162          * STFactory.java and PermanentVote.java.
163          */
164         public static final int PERMANENT_VOTES = 1000;
165 
166         /**
167          * LOCKING_VOTES is used (nominally by ADMIN voter, but not really by someone logged in as
168          * ADMIN, instead by combination of two PERMANENT_VOTES) to "lock" locale+path permanently
169          * in the LOCKED_XPATHS table. It is public for STFactory.PerLocaleData.loadVoteValues.
170          */
171         public static final int LOCKING_VOTES = 2000;
172 
173         /** The vote count a user of this level normally votes with */
174         private final int votes;
175 
176         /** The vote count a user of this level normally votes with if a tc org */
177         private final int tcorgvotes;
178 
179         /** The level as an integer, where 0 = admin, ..., 999 = locked */
180         private final int stlevel;
181 
Level(int votes, int stlevel, int tcorgvotes)182         Level(int votes, int stlevel, int tcorgvotes) {
183             this.votes = votes;
184             this.stlevel = stlevel;
185             this.tcorgvotes = tcorgvotes;
186         }
187 
Level(int votes, int stlevel)188         Level(int votes, int stlevel) {
189             this(votes, stlevel, votes);
190         }
191 
192         /**
193          * Get the votes for each level and organization
194          *
195          * @param o the given organization
196          */
getVotes(Organization o)197         public int getVotes(Organization o) {
198             if (this == vetter && o.isTCOrg()) {
199                 return tcorgvotes;
200             }
201             return votes;
202         }
203 
204         /** Get the Survey Tool userlevel for each level. (0=admin, 999=locked) */
getSTLevel()205         public int getSTLevel() {
206             return stlevel;
207         }
208 
209         /**
210          * Find the Level, given ST Level
211          *
212          * @param stlevel
213          * @return the Level corresponding to the integer
214          */
fromSTLevel(int stlevel)215         public static Level fromSTLevel(int stlevel) {
216             for (Level l : Level.values()) {
217                 if (l.getSTLevel() == stlevel) {
218                     return l;
219                 }
220             }
221             return null;
222         }
223 
224         /**
225          * Policy: can this user manage the "other" user's settings?
226          *
227          * @param myOrg the current organization
228          * @param otherLevel the other user's level
229          * @param otherOrg the other user's organization
230          * @return
231          */
isManagerFor(Organization myOrg, Level otherLevel, Organization otherOrg)232         public boolean isManagerFor(Organization myOrg, Level otherLevel, Organization otherOrg) {
233             return (this == admin
234                     || (canManageSomeUsers()
235                             && (myOrg == otherOrg)
236                             && atLeastAsPowerfulAs(otherLevel)));
237         }
238 
239         /**
240          * Policy: Can this user manage any users?
241          *
242          * @return
243          */
canManageSomeUsers()244         public boolean canManageSomeUsers() {
245             return atLeastAsPowerfulAs(manager);
246         }
247 
248         /** Internal: uses the ST Level as a measure of 'power' */
morePowerfulThan(Level other)249         boolean morePowerfulThan(Level other) {
250             return getSTLevel() < other.getSTLevel();
251         }
252 
253         /** Internal: uses the ST Level as a measure of 'power' */
atLeastAsPowerfulAs(Level other)254         boolean atLeastAsPowerfulAs(Level other) {
255             return getSTLevel() <= other.getSTLevel();
256         }
257 
258         /**
259          * Policy: can this user create or set a user to the specified level?
260          *
261          * @param otherLevel the desired new level for the other user
262          *     <p>Note: UserRegistry.canSetUserLevel enforces additional limitations depending on
263          *     more than this user's level and the other user's desired new level
264          */
canCreateOrSetLevelTo(Level otherLevel)265         public boolean canCreateOrSetLevelTo(Level otherLevel) {
266             // Must be a manager at all
267             if (!canManageSomeUsers()) return false;
268             // Cannot elevate privilege
269             return !otherLevel.morePowerfulThan(this);
270         }
271 
272         /**
273          * Can a user with this level and organization vote with the given vote count?
274          *
275          * @param org the given organization
276          * @param withVotes the given vote count
277          * @return true if the user can vote with the given vote count, else false
278          */
canVoteWithCount(Organization org, int withVotes)279         public boolean canVoteWithCount(Organization org, int withVotes) {
280             /*
281              * ADMIN is allowed to vote with LOCKING_VOTES, but not directly in the GUI, only
282              * by two TC voting together with PERMANENT_VOTES. Therefore LOCKING_VOTES is omitted
283              * from the GUI menu (voteCountMenu), but included in canVoteWithCount.
284              */
285             if (withVotes == LOCKING_VOTES && this == admin) {
286                 return true;
287             }
288             Set<Integer> menu = getVoteCountMenu(org);
289             return menu == null ? withVotes == getVotes(org) : menu.contains(withVotes);
290         }
291 
292         /**
293          * If not null, an array of different vote counts from which a user of this level is allowed
294          * to choose.
295          */
296         private ImmutableSet<Integer> voteCountMenu = null;
297 
298         /**
299          * Get the ordered immutable set of different vote counts a user of this level can vote with
300          *
301          * @param ignoredOrg the given organization
302          * @return the set, or null if the user has no choice of vote count
303          */
getVoteCountMenu(Organization ignoredOrg)304         public ImmutableSet<Integer> getVoteCountMenu(Organization ignoredOrg) {
305             // Right now, the organization does not affect the menu.
306             // but update the API to future proof.
307             return voteCountMenu;
308         }
309 
310         /*
311          * Set voteCountMenu for admin and tc in this static block, which will be run after
312          * all the constructors have run, rather than in the constructor itself. For example,
313          * vetter.votes needs to be defined before we can set admin.voteCountMenu.
314          */
315         static {
316             admin.voteCountMenu =
317                     ImmutableSet.of(
318                             guest.votes,
319                             vetter.votes,
320                             vetter.tcorgvotes,
321                             tc.votes,
322                             admin.votes,
323                             PERMANENT_VOTES);
324             /* Not LOCKING_VOTES; see canVoteWithCount */
325             tc.voteCountMenu =
326                     ImmutableSet.of(
327                             guest.votes,
328                             vetter.votes,
329                             vetter.tcorgvotes,
330                             tc.votes,
331                             PERMANENT_VOTES);
332         }
333 
334         // The following methods were moved here from UserRegistry
335         // TODO: remove this todo notice
336 
isAdmin()337         public boolean isAdmin() {
338             return stlevel <= admin.stlevel;
339         }
340 
isTC()341         public boolean isTC() {
342             return stlevel <= tc.stlevel;
343         }
344 
isExactlyManager()345         public boolean isExactlyManager() {
346             return stlevel == manager.stlevel;
347         }
348 
isManagerOrStronger()349         public boolean isManagerOrStronger() {
350             return stlevel <= manager.stlevel;
351         }
352 
isVetter()353         public boolean isVetter() {
354             return stlevel <= vetter.stlevel;
355         }
356 
isGuest()357         public boolean isGuest() {
358             return stlevel <= guest.stlevel;
359         }
360 
isLocked()361         public boolean isLocked() {
362             return stlevel == locked.stlevel;
363         }
364 
isExactlyAnonymous()365         public boolean isExactlyAnonymous() {
366             return stlevel == anonymous.stlevel;
367         }
368 
369         /**
370          * Is this user an administrator 'over' this user? Always true if admin, or if TC in same
371          * org.
372          *
373          * @param myOrg
374          */
isAdminForOrg(Organization myOrg, Organization target)375         public boolean isAdminForOrg(Organization myOrg, Organization target) {
376             return isAdmin() || ((isTC() || stlevel == manager.stlevel) && (myOrg == target));
377         }
378 
canImportOldVotes(CheckCLDR.Phase inPhase)379         public boolean canImportOldVotes(CheckCLDR.Phase inPhase) {
380             return isVetter() && (inPhase == Phase.SUBMISSION);
381         }
382 
canDoList()383         public boolean canDoList() {
384             return isVetter();
385         }
386 
canCreateUsers()387         public boolean canCreateUsers() {
388             return isTC() || isExactlyManager();
389         }
390 
canEmailUsers()391         public boolean canEmailUsers() {
392             return isTC() || isExactlyManager();
393         }
394 
canModifyUsers()395         public boolean canModifyUsers() {
396             return isTC() || isExactlyManager();
397         }
398 
canCreateOtherOrgs()399         public boolean canCreateOtherOrgs() {
400             return isAdmin();
401         }
402 
canUseVettingSummary()403         public boolean canUseVettingSummary() {
404             return isManagerOrStronger();
405         }
406 
canSubmit(CheckCLDR.Phase inPhase)407         public boolean canSubmit(CheckCLDR.Phase inPhase) {
408             if (inPhase == Phase.FINAL_TESTING) {
409                 return false;
410                 // TODO: Note, this will mean not just READONLY, but VETTING_CLOSED will return
411                 // false here.
412                 // This is probably desired!
413             }
414             return isGuest();
415         }
416 
canCreateSummarySnapshot()417         public boolean canCreateSummarySnapshot() {
418             return isAdmin();
419         }
420 
canMonitorForum()421         public boolean canMonitorForum() {
422             return isTC() || isExactlyManager();
423         }
424 
canSetInterestLocales()425         public boolean canSetInterestLocales() {
426             return isManagerOrStronger();
427         }
428 
canGetEmailList()429         public boolean canGetEmailList() {
430             return isManagerOrStronger();
431         }
432 
433         /** If true, can delete users at their user level or lower. */
canDeleteUsers()434         public boolean canDeleteUsers() {
435             return isAdmin();
436         }
437     }
438 
439     /**
440      * See getStatusForOrganization to see how this is computed.
441      *
442      * <p>Not to be confused with VoteResolver.Status
443      */
444     public enum VoteStatus {
445         /**
446          * The value for the path is either contributed or approved, and the user's organization
447          * didn't vote.
448          */
449         ok_novotes,
450 
451         /**
452          * The value for the path is either contributed or approved, and the user's organization
453          * chose the winning value.
454          */
455         ok,
456 
457         /** The winning value is neither contributed nor approved. */
458         provisionalOrWorse,
459 
460         /**
461          * The user's organization's choice is not winning, and the winning value is either
462          * contributed or approved. There may be insufficient votes to overcome a previously
463          * approved value, or other organizations may be voting against it.
464          */
465         losing,
466 
467         /**
468          * There is a dispute, meaning more than one item with votes, or the item with votes didn't
469          * win.
470          */
471         disputed
472     }
473 
474     /** Internal class for voter information. It is public for testing only */
475     public static class VoterInfo {
476         private Organization organization;
477         private Level level;
478         private String name;
479         /**
480          * A set of locales associated with this voter; it is often empty (as when the user has "*"
481          * for their set of locales); it may not serve any purpose in ordinary operation of Survey
482          * Tool; its main (only?) purpose seems to be for computeMaxVotes, whose only purpose seems
483          * to be creation of localeToOrganizationToMaxVote, which is used only by ConsoleCheckCLDR
484          * (for obscure reason), not by Survey Tool itself.
485          */
486         private final Set<CLDRLocale> locales = new TreeSet<>();
487 
getLocales()488         public Iterable<CLDRLocale> getLocales() {
489             return locales;
490         }
491 
VoterInfo(Organization organization, Level level, String name, LocaleSet localeSet)492         public VoterInfo(Organization organization, Level level, String name, LocaleSet localeSet) {
493             this.setOrganization(organization);
494             this.setLevel(level);
495             this.setName(name);
496             if (!localeSet.isAllLocales()) {
497                 this.locales.addAll(localeSet.getSet());
498             }
499         }
500 
VoterInfo(Organization organization, Level level, String name)501         public VoterInfo(Organization organization, Level level, String name) {
502             this.setOrganization(organization);
503             this.setLevel(level);
504             this.setName(name);
505         }
506 
VoterInfo()507         public VoterInfo() {}
508 
509         @Override
toString()510         public String toString() {
511             return "{" + getName() + ", " + getLevel() + ", " + getOrganization() + "}";
512         }
513 
setOrganization(Organization organization)514         public void setOrganization(Organization organization) {
515             this.organization = organization;
516         }
517 
getOrganization()518         public Organization getOrganization() {
519             return organization;
520         }
521 
setLevel(Level level)522         public void setLevel(Level level) {
523             this.level = level;
524         }
525 
getLevel()526         public Level getLevel() {
527             return level;
528         }
529 
setName(String name)530         public void setName(String name) {
531             this.name = name;
532         }
533 
getName()534         public String getName() {
535             return name;
536         }
537 
addLocale(CLDRLocale locale)538         void addLocale(CLDRLocale locale) {
539             this.locales.add(locale);
540         }
541 
542         @Override
equals(Object obj)543         public boolean equals(Object obj) {
544             if (obj == null) {
545                 return false;
546             }
547             VoterInfo other = (VoterInfo) obj;
548             return organization.equals(other.organization)
549                     && level.equals(other.level)
550                     && name.equals(other.name)
551                     && Objects.equal(locales, other.locales);
552         }
553 
554         @Override
hashCode()555         public int hashCode() {
556             return organization.hashCode() ^ level.hashCode() ^ name.hashCode();
557         }
558     }
559 
560     /**
561      * MaxCounter: make sure that we are always only getting the maximum of the values.
562      *
563      * @author markdavis
564      * @param <T>
565      */
566     static class MaxCounter<T> extends Counter<T> {
MaxCounter(boolean b)567         public MaxCounter(boolean b) {
568             super(b);
569         }
570 
571         /** Add, but only to bring up to the maximum value. */
572         @Override
add(T obj, long countValue, long time)573         public MaxCounter<T> add(T obj, long countValue, long time) {
574             long value = getCount(obj);
575             if ((value <= countValue)) {
576                 super.add(obj, countValue - value, time); // only add the difference!
577             }
578             return this;
579         }
580     }
581 
582     /** Internal class for getting from an organization to its vote. */
583     private class OrganizationToValueAndVote<T> {
584         private final Map<Organization, MaxCounter<T>> orgToVotes =
585                 new EnumMap<>(Organization.class);
586         /**
587          * All votes, even those that aren't any org's vote because they lost an intra-org dispute
588          */
589         private final Counter<T> allVotesIncludingIntraOrgDispute = new Counter<>();
590 
591         private final Map<Organization, Integer> orgToMax = new EnumMap<>(Organization.class);
592         /** The result of {@link #getTotals(EnumSet)} */
593         private final Counter<T> totals = new Counter<>(true);
594 
595         private final Map<String, Long> nameTime = new LinkedHashMap<>();
596 
597         /** map an organization to the value it voted for. */
598         private final Map<Organization, T> orgToAdd = new EnumMap<>(Organization.class);
599 
600         private T baileyValue;
601         private boolean baileySet; // was the bailey value set
602 
OrganizationToValueAndVote()603         OrganizationToValueAndVote() {
604             for (Organization org : Organization.values()) {
605                 orgToVotes.put(org, new MaxCounter<>(true));
606             }
607         }
608 
609         /** Call clear before considering each new path */
clear()610         public void clear() {
611             for (Map.Entry<Organization, MaxCounter<T>> entry : orgToVotes.entrySet()) {
612                 entry.getValue().clear();
613             }
614             orgToAdd.clear();
615             orgToMax.clear();
616             allVotesIncludingIntraOrgDispute.clear();
617             baileyValue = null;
618             baileySet = false;
619             if (transcript != null) {
620                 // there was a transcript before, so retain it
621                 transcript = new DeferredTranscript();
622             }
623         }
624 
getNameTime()625         public Map<String, Long> getNameTime() {
626             return nameTime;
627         }
628 
629         /**
630          * Call this to add votes
631          *
632          * @param value
633          * @param voter
634          * @param withVotes optionally, vote at a non-typical voting level. May not exceed voter's
635          *     maximum allowed level. null = use default level.
636          * @param date
637          */
add(T value, int voter, Integer withVotes, Date date)638         public void add(T value, int voter, Integer withVotes, Date date) {
639             final VoterInfo info = voterInfoList.get(voter);
640             if (info == null) {
641                 throw new UnknownVoterException(voter);
642             }
643             Level level = info.getLevel();
644             if (withVotes == null || !level.canVoteWithCount(info.organization, withVotes)) {
645                 withVotes = level.getVotes(info.organization);
646             }
647             addInternal(value, info, withVotes, date); // do the add
648         }
649 
650         /**
651          * Called by add(T,int,Integer) to actually add a value.
652          *
653          * @param value
654          * @param info
655          * @param votes
656          * @param time
657          * @see #add(Object, int, Integer)
658          */
addInternal(T value, final VoterInfo info, final int votes, Date time)659         private void addInternal(T value, final VoterInfo info, final int votes, Date time) {
660             if (DROP_HARD_INHERITANCE) {
661                 value = changeBaileyToInheritance(value);
662             }
663             /* All votes are added here, even if they will later lose an intra-org dispute. */
664             allVotesIncludingIntraOrgDispute.add(value, votes, time.getTime());
665             nameTime.put(info.getName(), time.getTime());
666             if (DEBUG) {
667                 System.out.println(
668                         "allVotesIncludingIntraOrgDispute Info: "
669                                 + allVotesIncludingIntraOrgDispute);
670             }
671             if (DEBUG) {
672                 System.out.println("VoteInfo: " + info.getName() + info.getOrganization());
673             }
674             Organization organization = info.getOrganization();
675             orgToVotes.get(organization).add(value, votes, time.getTime());
676             if (DEBUG) {
677                 System.out.println(
678                         "Adding now Info: "
679                                 + organization.getDisplayName()
680                                 + info.getName()
681                                 + " is adding: "
682                                 + votes
683                                 + value
684                                 + new Timestamp(time.getTime()));
685             }
686 
687             if (DEBUG) {
688                 System.out.println(
689                         "addInternal: "
690                                 + organization.getDisplayName()
691                                 + " : "
692                                 + orgToVotes.get(organization).toString());
693             }
694 
695             // add the new votes to orgToMax, if they are greater that what was there
696             Integer max = orgToMax.get(info.getOrganization());
697             if (max == null || max < votes) {
698                 orgToMax.put(organization, votes);
699             }
700         }
701 
702         /**
703          * Return the overall vote for each organization. It is the max for each value. When the
704          * organization is conflicted (the top two values have the same vote), the organization is
705          * also added to disputed.
706          *
707          * @param conflictedOrganizations if not null, to be filled in with the set of conflicted
708          *     organizations.
709          */
getTotals(EnumSet<Organization> conflictedOrganizations)710         public Counter<T> getTotals(EnumSet<Organization> conflictedOrganizations) {
711             if (conflictedOrganizations != null) {
712                 conflictedOrganizations.clear();
713             }
714             totals.clear();
715 
716             annotateTranscript("- Getting all totals by organization:");
717             for (Map.Entry<Organization, MaxCounter<T>> entry : orgToVotes.entrySet()) {
718                 Counter<T> items = entry.getValue();
719                 if (items.size() == 0) {
720                     continue;
721                 }
722                 Iterator<T> iterator = items.getKeysetSortedByCount(false).iterator();
723                 T value = iterator.next();
724                 long weight = items.getCount(value);
725                 if (weight == 0) {
726                     continue;
727                 }
728                 annotateTranscript(
729                         "-- Considering %s which has %d item(s)",
730                         entry.getKey().getDisplayName(), items.size());
731                 Organization org = entry.getKey();
732                 if (DEBUG) {
733                     System.out.println("sortedKeys?? " + value + " " + org.getDisplayName());
734                 }
735                 // if there is more than one item, check that it is less
736                 if (iterator.hasNext()) {
737                     T value2 = iterator.next();
738                     long weight2 = items.getCount(value2);
739                     // if the votes for #1 are not better than #2, we have a dispute
740                     if (weight == weight2) {
741                         if (conflictedOrganizations != null) {
742                             annotateTranscript(
743                                     "--- There are conflicts due to different values by users of this organization.");
744                             conflictedOrganizations.add(org);
745                         }
746                     }
747                 }
748                 // This is deprecated, but preserve it until the method is removed.
749                 /*
750                  * TODO: explain the above comment, and follow through. What is deprecated (orgToAdd, or getOrgVote)?
751                  * Preserve until which method is removed (getOrgVote)?
752                  */
753                 orgToAdd.put(org, value);
754 
755                 // We add the max vote for each of the organizations choices
756                 long maxCount = 0;
757                 T considerItem = null;
758                 long considerCount = 0;
759                 long maxtime = 0;
760                 long considerTime = 0;
761                 for (T item : items.keySet()) {
762                     if (DEBUG) {
763                         System.out.println(
764                                 "Items in order: "
765                                         + item.toString()
766                                         + new Timestamp(items.getTime(item)));
767                     }
768                     long count = items.getCount(item);
769                     long time = items.getTime(item);
770                     if (count > maxCount) {
771                         maxCount = count;
772                         maxtime = time;
773                         // tell the 'losing' item
774                         if (considerItem != null) {
775                             annotateTranscript(
776                                     "---- Org is not voting for '%s': there is a higher ranked vote",
777                                     considerItem);
778                         }
779                         considerItem = item;
780                         if (DEBUG) {
781                             System.out.println(
782                                     "count>maxCount: "
783                                             + considerItem
784                                             + ":"
785                                             + new Timestamp(considerTime)
786                                             + " COUNT: "
787                                             + considerCount
788                                             + "MAXCOUNT: "
789                                             + maxCount);
790                         }
791                         considerCount = items.getCount(considerItem);
792                         considerTime = items.getTime(considerItem);
793                     } else if ((time > maxtime) && (count == maxCount)) {
794                         maxtime = time;
795                         // tell the 'losing' item
796                         if (considerItem != null) {
797                             annotateTranscript(
798                                     "---- Org is not voting for '%s': there is a later vote",
799                                     considerItem);
800                         }
801                         considerItem = item;
802                         considerCount = items.getCount(considerItem);
803                         considerTime = items.getTime(considerItem);
804                         if (DEBUG) {
805                             System.out.println(
806                                     "time>maxTime: "
807                                             + considerItem
808                                             + ":"
809                                             + new Timestamp(considerTime));
810                         }
811                     }
812                 }
813                 annotateTranscript(
814                         "--- %s vote is for '%s' with strength %d",
815                         org.getDisplayName(), considerItem, considerCount);
816                 orgToAdd.put(org, considerItem);
817                 totals.add(considerItem, considerCount, considerTime);
818 
819                 if (DEBUG) {
820                     System.out.println("Totals: " + totals + " : " + new Timestamp(considerTime));
821                 }
822             }
823 
824             if (DEBUG) {
825                 System.out.println("FINALTotals: " + totals);
826             }
827             return totals;
828         }
829 
getOrgCount(T winningValue)830         public int getOrgCount(T winningValue) {
831             int orgCount = 0;
832             for (Map.Entry<Organization, MaxCounter<T>> entry : orgToVotes.entrySet()) {
833                 Counter<T> counter = entry.getValue();
834                 long count = counter.getCount(winningValue);
835                 if (count > 0) {
836                     orgCount++;
837                 }
838             }
839             return orgCount;
840         }
841 
getBestPossibleVote()842         private int getBestPossibleVote() {
843             int total = 0;
844             for (Map.Entry<Organization, Integer> entry : orgToMax.entrySet()) {
845                 total += entry.getValue();
846             }
847             return total;
848         }
849 
850         @Override
toString()851         public String toString() {
852             String orgToVotesString = "";
853             for (Entry<Organization, MaxCounter<T>> entry : orgToVotes.entrySet()) {
854                 Counter<T> counter = entry.getValue();
855                 if (counter.size() != 0) {
856                     if (orgToVotesString.length() != 0) {
857                         orgToVotesString += ", ";
858                     }
859                     Organization org = entry.getKey();
860                     orgToVotesString += org.toString() + "=" + counter;
861                 }
862             }
863             EnumSet<Organization> conflicted = EnumSet.noneOf(Organization.class);
864             return "{orgToVotes: "
865                     + orgToVotesString
866                     + ", totals: "
867                     + getTotals(conflicted)
868                     + ", conflicted: "
869                     + conflicted
870                     + "}";
871         }
872 
873         /**
874          * This is now deprecated, since the organization may have multiple votes.
875          *
876          * @param org
877          * @return
878          * @deprecated
879          */
880         @Deprecated
getOrgVote(Organization org)881         public T getOrgVote(Organization org) {
882             return orgToAdd.get(org);
883         }
884 
getOrgVoteRaw(Organization orgOfUser)885         public T getOrgVoteRaw(Organization orgOfUser) {
886             return orgToAdd.get(orgOfUser);
887         }
888 
getOrgToVotes(Organization org)889         public Map<T, Long> getOrgToVotes(Organization org) {
890             Map<T, Long> result = new LinkedHashMap<>();
891             MaxCounter<T> counter = orgToVotes.get(org);
892             for (T item : counter) {
893                 result.put(item, counter.getCount(item));
894             }
895             return result;
896         }
897     }
898 
899     /** Data built internally */
900     private T winningValue;
901 
902     private T oValue; // optimal value; winning if better approval status than old
903     private T nValue; // next to optimal value
904     private final List<T> valuesWithSameVotes = new ArrayList<>();
905     private Counter<T> totals = null;
906 
907     private Status winningStatus;
908     private final EnumSet<Organization> conflictedOrganizations =
909             EnumSet.noneOf(Organization.class);
910     private final OrganizationToValueAndVote<T> organizationToValueAndVote =
911             new OrganizationToValueAndVote<>();
912     private T baselineValue;
913     private Status baselineStatus;
914 
915     private boolean resolved;
916     private boolean valueIsLocked;
917     private int requiredVotes = 0;
918     private final SupplementalDataInfo supplementalDataInfo = SupplementalDataInfo.getInstance();
919     private CLDRLocale locale;
920     private PathHeader pathHeader;
921 
922     private static final Collator englishCollator = Collator.getInstance(ULocale.ENGLISH).freeze();
923 
924     /** Used for comparing objects of type T */
925     private final Comparator<T> objectCollator =
926             (o1, o2) -> englishCollator.compare(String.valueOf(o1), String.valueOf(o2));
927 
928     /**
929      * Set the baseline (or "trunk") value and status for this VoteResolver.
930      *
931      * @param baselineValue the baseline value
932      * @param baselineStatus the baseline status
933      */
setBaseline(T baselineValue, Status baselineStatus)934     public void setBaseline(T baselineValue, Status baselineStatus) {
935         this.baselineValue = baselineValue;
936         this.baselineStatus = baselineValue == null ? Status.missing : baselineStatus;
937     }
938 
getBaselineValue()939     public T getBaselineValue() {
940         return baselineValue;
941     }
942 
getBaselineStatus()943     public Status getBaselineStatus() {
944         return baselineStatus;
945     }
946 
947     /**
948      * Set the locale and PathHeader for this VoteResolver
949      *
950      * <p>You must call this whenever you are using a VoteResolver with a new locale or a new
951      * PathHeader
952      *
953      * @param locale the CLDRLocale
954      * @param pathHeader the PathHeader
955      */
setLocale(CLDRLocale locale, PathHeader pathHeader)956     public void setLocale(CLDRLocale locale, PathHeader pathHeader) {
957         this.locale = locale;
958         this.pathHeader = pathHeader;
959     }
960 
961     /**
962      * What are the required votes for this item?
963      *
964      * @return the number of votes (as of this writing: usually 4, 8 for established locales)
965      */
getRequiredVotes()966     public int getRequiredVotes() {
967         if (requiredVotes == 0) {
968             int preliminaryRequiredVotes =
969                     supplementalDataInfo.getRequiredVotes(locale, pathHeader);
970             if (preliminaryRequiredVotes == HIGH_BAR && baselineStatus != Status.approved) {
971                 requiredVotes = LOWER_BAR;
972             } else {
973                 requiredVotes = preliminaryRequiredVotes;
974             }
975         }
976         return requiredVotes;
977     }
978 
979     /**
980      * Call this method first, for a new base path. You'll then call add for each value associated
981      * with that base path.
982      */
clear()983     public void clear() {
984         baselineValue = null;
985         baselineStatus = Status.missing;
986         requiredVotes = 0;
987         locale = null;
988         pathHeader = null;
989         organizationToValueAndVote.clear();
990         resolved = valueIsLocked = false;
991         values.clear();
992 
993         // TODO: clear these out between reuse
994         // Are there other values that should be cleared?
995         oValue = null;
996         setWinningValue(null);
997         nValue = null;
998 
999         if (transcript != null) {
1000             transcript.clear();
1001         }
1002     }
1003 
1004     /**
1005      * Get the bailey value (what the inherited value would be if there were no explicit value) for
1006      * this VoteResolver.
1007      *
1008      * <p>Throw an exception if !baileySet.
1009      *
1010      * @return the bailey value.
1011      *     <p>Called by STFactory.PerLocaleData.getResolverInternal in the special circumstance
1012      *     where getWinningValue has returned INHERITANCE_MARKER.
1013      */
getBaileyValue()1014     public T getBaileyValue() {
1015         if (!organizationToValueAndVote.baileySet) {
1016             throw new IllegalArgumentException(
1017                     "setBaileyValue must be called before getBaileyValue");
1018         }
1019         return organizationToValueAndVote.baileyValue;
1020     }
1021 
1022     /**
1023      * Set the Bailey value (what the inherited value would be if there were no explicit value).
1024      * This value is used in handling any CldrUtility.INHERITANCE_MARKER. This value must be set
1025      * <i>before</i> adding values. Usually by calling CLDRFile.getBaileyValue().
1026      */
setBaileyValue(T baileyValue)1027     public void setBaileyValue(T baileyValue) {
1028         organizationToValueAndVote.baileySet = true;
1029         organizationToValueAndVote.baileyValue = baileyValue;
1030     }
1031 
1032     /**
1033      * Call once for each voter for a value. If there are no voters for an item, then call
1034      * add(value);
1035      *
1036      * @param value
1037      * @param voter
1038      * @param withVotes override to lower the user's voting permission. May be null for default.
1039      * @param date
1040      *     <p>Called by getResolverInternal in STFactory, and elsewhere
1041      */
add(T value, int voter, Integer withVotes, Date date)1042     public void add(T value, int voter, Integer withVotes, Date date) {
1043         if (DROP_HARD_INHERITANCE) {
1044             value = changeBaileyToInheritance(value);
1045         }
1046         if (resolved) {
1047             throw new IllegalArgumentException(
1048                     "Must be called after clear, and before any getters.");
1049         }
1050         if (withVotes != null && withVotes == Level.LOCKING_VOTES) {
1051             valueIsLocked = true;
1052         }
1053         organizationToValueAndVote.add(value, voter, withVotes, date);
1054         values.add(value);
1055     }
1056 
1057     /**
1058      * Call once for each voter for a value. If there are no voters for an item, then call
1059      * add(value);
1060      *
1061      * @param value
1062      * @param voter
1063      * @param withVotes override to lower the user's voting permission. May be null for default.
1064      *     <p>Called only for TestUtilities, not used in Survey Tool.
1065      */
add(T value, int voter, Integer withVotes)1066     public void add(T value, int voter, Integer withVotes) {
1067         if (DROP_HARD_INHERITANCE) {
1068             value = changeBaileyToInheritance(value);
1069         }
1070         if (resolved) {
1071             throw new IllegalArgumentException(
1072                     "Must be called after clear, and before any getters.");
1073         }
1074         Date date = new Date();
1075         organizationToValueAndVote.add(value, voter, withVotes, date);
1076         values.add(value);
1077     }
1078 
changeBaileyToInheritance(T value)1079     private <T> T changeBaileyToInheritance(T value) {
1080         if (value != null && value.equals(getBaileyValue())) {
1081             return (T) CldrUtility.INHERITANCE_MARKER;
1082         }
1083         return value;
1084     }
1085 
1086     /** Used only in add(value, voter) for making a pseudo-Date */
1087     private int maxcounter = 100;
1088 
1089     /**
1090      * Call once for each voter for a value. If there are no voters for an item, then call
1091      * add(value);
1092      *
1093      * @param value
1094      * @param voter
1095      *     <p>Called by ConsoleCheckCLDR and TestUtilities; not used in SurveyTool.
1096      */
add(T value, int voter)1097     public void add(T value, int voter) {
1098         Date date = new Date(++maxcounter);
1099         add(value, voter, null, date);
1100     }
1101 
1102     /**
1103      * Call if a value has no voters. It is safe to also call this if there is a voter, just
1104      * unnecessary.
1105      *
1106      * @param value
1107      *     <p>Called by getResolverInternal for the baseline (trunk) value; also called for
1108      *     ConsoleCheckCLDR.
1109      */
add(T value)1110     public void add(T value) {
1111         if (resolved) {
1112             throw new IllegalArgumentException(
1113                     "Must be called after clear, and before any getters.");
1114         }
1115         values.add(value);
1116     }
1117 
1118     private final Set<T> values = new TreeSet<>(objectCollator);
1119 
1120     private final Comparator<T> votesThenUcaCollator =
1121             new Comparator<>() {
1122 
1123                 /**
1124                  * Compare candidate items by vote count, highest vote first. In the case of ties,
1125                  * favor (a) the baseline (trunk) value, then (b) votes for inheritance
1126                  * (INHERITANCE_MARKER), then (c) the alphabetical order (as a last resort).
1127                  *
1128                  * <p>Return negative to favor o1, positive to favor o2.
1129                  *
1130                  * @see VoteResolver#setBestNextAndSameVoteValues(Set, HashMap)
1131                  * @see VoteResolver#annotateNextBestValue(long, long, T, T)
1132                  */
1133                 @Override
1134                 public int compare(T o1, T o2) {
1135                     long v1 = organizationToValueAndVote.allVotesIncludingIntraOrgDispute.get(o1);
1136                     long v2 = organizationToValueAndVote.allVotesIncludingIntraOrgDispute.get(o2);
1137                     if (v1 != v2) {
1138                         return v1 < v2 ? 1 : -1; // highest vote first
1139                     }
1140                     if (o1.equals(baselineValue)) {
1141                         return -1;
1142                     } else if (o2.equals(baselineValue)) {
1143                         return 1;
1144                     }
1145                     if (o1.equals(CldrUtility.INHERITANCE_MARKER)) {
1146                         return -1;
1147                     } else if (o2.equals(CldrUtility.INHERITANCE_MARKER)) {
1148                         return 1;
1149                     }
1150                     return englishCollator.compare(String.valueOf(o1), String.valueOf(o2));
1151                 }
1152             };
1153 
1154     /**
1155      * Annotate why the O (winning) value is winning vs the N (next) value. Assumes that the prior
1156      * annotation mentioned the O value.
1157      *
1158      * @param O optimal value
1159      * @param N next-best value
1160      */
annotateNextBestValue(long O, long N, final T oValue, final T nValue)1161     private void annotateNextBestValue(long O, long N, final T oValue, final T nValue) {
1162         // See the Comparator<> defined immediately above.
1163 
1164         // sortedValues.size() >= 2 - explain why O won and N lost.
1165         // We have to perform the function of the votesThenUcaCollator one more time
1166         if (O > N) {
1167             annotateTranscript(
1168                     "- This is the optimal value because it has the highest weight (voting score).");
1169         } else if (winningValue.equals(baselineValue)) {
1170             annotateTranscript(
1171                     "- This is the optimal value because it is the same as the baseline value, though the weight was otherwise equal to the next-best."); // aka blue star
1172         } else if (winningValue.equals(CldrUtility.INHERITANCE_MARKER)) {
1173             annotateTranscript(
1174                     "- This is the optimal value because it is the inheritance marker, though the weight was otherwise equal to the next-best."); // triple up arrow
1175         } else {
1176             annotateTranscript(
1177                     "- This is the optimal value because it comes earlier than '%s' when the text was sorted, though the weight was otherwise equal to the next-best.",
1178                     nValue);
1179         }
1180         annotateTranscript("The Next-best (N) value is '%s', with weight %d", nValue, N);
1181     }
1182 
1183     /** This will be changed to true if both kinds of vote are present */
1184     private boolean bothInheritanceAndBaileyHadVotes = false;
1185 
1186     /**
1187      * Resolve the votes. Resolution entails counting votes and setting members for this
1188      * VoteResolver, including winningStatus, winningValue, and many others.
1189      */
resolveVotes()1190     private void resolveVotes() {
1191         annotateTranscript("Resolving votes:");
1192         resolved = true;
1193         // get the votes for each organization
1194         valuesWithSameVotes.clear();
1195         totals = organizationToValueAndVote.getTotals(conflictedOrganizations);
1196         /* Note: getKeysetSortedByCount actually returns a LinkedHashSet, "with predictable iteration order". */
1197         final Set<T> sortedValues = totals.getKeysetSortedByCount(false, votesThenUcaCollator);
1198         if (DEBUG) {
1199             System.out.println("sortedValues :" + sortedValues.toString());
1200         }
1201         // annotateTranscript("all votes by org: %s", sortedValues);
1202 
1203         /*
1204          * If there are no (unconflicted) votes, return baseline (trunk) if not null,
1205          * else INHERITANCE_MARKER if baileySet, else NO_WINNING_VALUE.
1206          * Avoid setting winningValue to null. VoteResolver should be fully in charge of vote resolution.
1207          */
1208         if (sortedValues.size() == 0) {
1209             if (baselineValue != null) {
1210                 setWinningValue(baselineValue);
1211                 winningStatus = baselineStatus;
1212                 annotateTranscript(
1213                         "Winning Value: '%s' with status '%s' because there were no unconflicted votes.",
1214                         winningValue, winningStatus);
1215                 // Declare the winner here, because we're about to return from the function
1216             } else if (organizationToValueAndVote.baileySet) {
1217                 setWinningValue((T) CldrUtility.INHERITANCE_MARKER);
1218                 winningStatus = Status.missing;
1219                 annotateTranscript(
1220                         "Winning Value: '%s' with status '%s' because there were no unconflicted votes, and there was a Bailey value set.",
1221                         winningValue, winningStatus);
1222                 // Declare the winner here, because we're about to return from the function
1223             } else {
1224                 /*
1225                  * TODO: When can this still happen? See https://unicode.org/cldr/trac/ticket/11299 "Example C".
1226                  * Also http://localhost:8080/cldr-apps/v#/en_CA/Gregorian/
1227                  * -- also http://localhost:8080/cldr-apps/v#/aa/Languages_A_D/
1228                  *    xpath //ldml/localeDisplayNames/languages/language[@type="zh_Hans"][@alt="long"]
1229                  * See also checkDataRowConsistency in DataSection.java.
1230                  */
1231                 setWinningValue((T) NO_WINNING_VALUE);
1232                 winningStatus = Status.missing;
1233                 annotateTranscript(
1234                         "No winning value! status '%s' because there were no unconflicted votes",
1235                         winningStatus);
1236                 // Declare the non-winner here, because we're about to return from the function
1237             }
1238             valuesWithSameVotes.add(winningValue);
1239             return; // sortedValues.size() == 0, no candidates
1240         }
1241         if (values.size() == 0) {
1242             throw new IllegalArgumentException("No values added to resolver");
1243         }
1244 
1245         /*
1246          * Copy what is in the the totals field of this VoteResolver for all the
1247          * values in sortedValues. This local variable voteCount may be used
1248          * subsequently to make adjustments for vote resolution. Those adjustment
1249          * may affect the winners in vote resolution, while still preserving the original
1250          * voting data including the totals field.
1251          */
1252         HashMap<T, Long> voteCount = makeVoteCountMap(sortedValues);
1253 
1254         /*
1255          * Adjust sortedValues and voteCount as needed to combine "soft" votes for inheritance
1256          * with "hard" votes for the Bailey value. Note that sortedValues and voteCount are
1257          * both local variables.
1258          */
1259         if (!DROP_HARD_INHERITANCE) {
1260             bothInheritanceAndBaileyHadVotes =
1261                     combineInheritanceWithBaileyForVoting(sortedValues, voteCount);
1262         }
1263 
1264         /*
1265          * Adjust sortedValues and voteCount as needed for annotation keywords.
1266          */
1267         if (isUsingKeywordAnnotationVoting()) {
1268             adjustAnnotationVoteCounts(sortedValues, voteCount);
1269         }
1270 
1271         /*
1272          * Perform the actual resolution.
1273          * This sets winningValue to the top element of
1274          * sortedValues.
1275          */
1276         long[] weights = setBestNextAndSameVoteValues(sortedValues, voteCount);
1277 
1278         oValue = winningValue;
1279 
1280         winningStatus = computeStatus(weights[0], weights[1]);
1281 
1282         // if we are not as good as the baseline (trunk), use the baseline
1283         // TODO: how could baselineStatus be null here??
1284         if (baselineStatus != null && winningStatus.compareTo(baselineStatus) < 0) {
1285             setWinningValue(baselineValue);
1286             annotateTranscript(
1287                     "The optimal value so far with status '%s' would not be as good as the baseline status. "
1288                             + "Therefore, the winning value is '%s' with status '%s'.",
1289                     winningStatus, winningValue, baselineStatus);
1290             winningStatus = baselineStatus;
1291             valuesWithSameVotes.clear();
1292             valuesWithSameVotes.add(winningValue);
1293         } else {
1294             // Declare the final winner
1295             annotateTranscript(
1296                     "The winning value is '%s' with status '%s'.", winningValue, winningStatus);
1297         }
1298     }
1299 
1300     /**
1301      * Make a hash for the vote count of each value in the given sorted list, using the totals field
1302      * of this VoteResolver.
1303      *
1304      * <p>This enables subsequent local adjustment of the effective votes, without change to the
1305      * totals field. Purposes include inheritance and annotation voting.
1306      *
1307      * @param sortedValues the sorted list of values (really a LinkedHashSet, "with predictable
1308      *     iteration order")
1309      * @return the HashMap
1310      */
makeVoteCountMap(Set<T> sortedValues)1311     private HashMap<T, Long> makeVoteCountMap(Set<T> sortedValues) {
1312         HashMap<T, Long> map = new HashMap<>();
1313         for (T value : sortedValues) {
1314             map.put(value, totals.getCount(value));
1315         }
1316         return map;
1317     }
1318 
1319     /**
1320      * Adjust the given sortedValues and voteCount, if necessary, to combine "hard" and "soft"
1321      * votes. Do nothing unless both hard and soft votes are present.
1322      *
1323      * <p>For voting resolution in which inheritance plays a role, "soft" votes for inheritance are
1324      * distinct from "hard" (explicit) votes for the Bailey value. For resolution, these two kinds
1325      * of votes are treated in combination. If that combination is winning, then the final winner
1326      * will be the hard item or the soft item, whichever has more votes, the soft item winning if
1327      * they're tied. Except for the soft item being favored as a tie-breaker, this function should
1328      * be symmetrical in its handling of hard and soft votes.
1329      *
1330      * <p>Note: now that "↑↑↑" is permitted to participate directly in voting resolution, it becomes
1331      * significant that with Collator.getInstance(ULocale.ENGLISH), "↑↑↑" sorts before "AAA" just as
1332      * "AAA" sorts before "BBB".
1333      *
1334      * @param sortedValues the set of sorted values, possibly to be modified
1335      * @param voteCount the hash giving the vote count for each value, possibly to be modified
1336      * @return true if both "hard" and "soft" votes existed and were combined, else false
1337      */
combineInheritanceWithBaileyForVoting( Set<T> sortedValues, HashMap<T, Long> voteCount)1338     private boolean combineInheritanceWithBaileyForVoting(
1339             Set<T> sortedValues, HashMap<T, Long> voteCount) {
1340         if (organizationToValueAndVote.baileySet == false
1341                 || organizationToValueAndVote.baileyValue == null) {
1342             return false;
1343         }
1344         T hardValue = organizationToValueAndVote.baileyValue;
1345         T softValue = (T) CldrUtility.INHERITANCE_MARKER;
1346         /*
1347          * Check containsKey before get, to avoid NullPointerException.
1348          */
1349         if (!voteCount.containsKey(hardValue) || !voteCount.containsKey(softValue)) {
1350             return false;
1351         }
1352         long hardCount = voteCount.get(hardValue);
1353         long softCount = voteCount.get(softValue);
1354         if (hardCount == 0 || softCount == 0) {
1355             return false;
1356         }
1357         reallyCombineInheritanceWithBailey(
1358                 sortedValues, voteCount, hardValue, softValue, hardCount, softCount);
1359         return true;
1360     }
1361 
1362     /**
1363      * Given that both "hard" and "soft" votes exist, combine them
1364      *
1365      * @param sortedValues the set of sorted values, to be modified
1366      * @param voteCount the hash giving the vote count for each value, to be modified
1367      * @param hardValue the bailey value
1368      * @param softValue the inheritance marker
1369      * @param hardCount the number of votes for hardValue
1370      * @param softCount the number of votes for softValue
1371      */
reallyCombineInheritanceWithBailey( Set<T> sortedValues, HashMap<T, Long> voteCount, T hardValue, T softValue, long hardCount, long softCount)1372     private void reallyCombineInheritanceWithBailey(
1373             Set<T> sortedValues,
1374             HashMap<T, Long> voteCount,
1375             T hardValue,
1376             T softValue,
1377             long hardCount,
1378             long softCount) {
1379         final T combValue = (hardCount > softCount) ? hardValue : softValue;
1380         final T skipValue = (hardCount > softCount) ? softValue : hardValue;
1381         final long combinedCount = hardCount + softCount;
1382         voteCount.put(combValue, combinedCount);
1383         voteCount.put(skipValue, 0L);
1384         /*
1385          * Sort again
1386          */
1387         List<T> list = new ArrayList<>(sortedValues);
1388         list.sort(
1389                 (v1, v2) -> {
1390                     long c1 = voteCount.get(v1);
1391                     long c2 = voteCount.get(v2);
1392                     if (c1 != c2) {
1393                         return (c1 < c2) ? 1 : -1; // decreasing numeric order (most votes wins)
1394                     }
1395                     return englishCollator.compare(String.valueOf(v1), String.valueOf(v2));
1396                 });
1397         /*
1398          * Omit skipValue
1399          */
1400         sortedValues.clear();
1401         for (T value : list) {
1402             if (!value.equals(skipValue)) {
1403                 sortedValues.add(value);
1404             }
1405         }
1406     }
1407 
1408     /**
1409      * Adjust the effective votes for bar-joined annotations, and re-sort the array of values to
1410      * reflect the adjusted vote counts.
1411      *
1412      * <p>Note: "Annotations provide names and keywords for Unicode characters, currently focusing
1413      * on emoji." For example, an annotation "happy | joyful" has two components "happy" and
1414      * "joyful". References: http://unicode.org/cldr/charts/32/annotations/index.html
1415      * https://www.unicode.org/reports/tr35/tr35-general.html#Annotations
1416      *
1417      * <p>http://unicode.org/repos/cldr/tags/latest/common/annotations/
1418      *
1419      * @param sortedValues the set of sorted values
1420      * @param voteCount the hash giving the vote count for each value in sortedValues
1421      *     <p>public for unit testing, see TestAnnotationVotes.java
1422      */
adjustAnnotationVoteCounts(Set<T> sortedValues, HashMap<T, Long> voteCount)1423     public void adjustAnnotationVoteCounts(Set<T> sortedValues, HashMap<T, Long> voteCount) {
1424         if (voteCount == null || sortedValues == null) {
1425             return;
1426         }
1427         annotateTranscript("Vote weights are being adjusted due to annotation keywords.");
1428 
1429         // Make compMap map individual components to cumulative vote counts.
1430         HashMap<T, Long> compMap = makeAnnotationComponentMap(sortedValues, voteCount);
1431 
1432         // Save a copy of the "raw" vote count before adjustment, since it's needed by
1433         // promoteSuperiorAnnotationSuperset.
1434         HashMap<T, Long> rawVoteCount = new HashMap<>(voteCount);
1435 
1436         // Calculate new counts for original values, based on components.
1437         calculateNewCountsBasedOnAnnotationComponents(sortedValues, voteCount, compMap);
1438 
1439         // Re-sort sortedValues based on voteCount.
1440         resortValuesBasedOnAdjustedVoteCounts(sortedValues, voteCount);
1441 
1442         // If the set that so far is winning has supersets with superior raw vote count, promote the
1443         // supersets.
1444         promoteSuperiorAnnotationSuperset(sortedValues, voteCount, rawVoteCount);
1445     }
1446 
1447     /**
1448      * Make a hash that maps individual annotation components to cumulative vote counts.
1449      *
1450      * <p>For example, 3 votes for "a|b" and 2 votes for "a|c" makes 5 votes for "a", 3 for "b", and
1451      * 2 for "c".
1452      *
1453      * @param sortedValues the set of sorted values
1454      * @param voteCount the hash giving the vote count for each value in sortedValues
1455      */
makeAnnotationComponentMap( Set<T> sortedValues, HashMap<T, Long> voteCount)1456     private HashMap<T, Long> makeAnnotationComponentMap(
1457             Set<T> sortedValues, HashMap<T, Long> voteCount) {
1458         HashMap<T, Long> compMap = new HashMap<>();
1459         annotateTranscript("- First, components are split up and total votes calculated");
1460         for (T value : sortedValues) {
1461             Long count = voteCount.get(value);
1462             List<T> comps = splitAnnotationIntoComponentsList(value);
1463             for (T comp : comps) {
1464                 if (compMap.containsKey(comp)) {
1465                     compMap.replace(comp, compMap.get(comp) + count);
1466                 } else {
1467                     compMap.put(comp, count);
1468                 }
1469             }
1470         }
1471         if (transcript != null && !DEBUG) {
1472             for (Entry<T, Long> comp : compMap.entrySet()) {
1473                 // TODO: could sort here, or not.
1474                 annotateTranscript(
1475                         "-- component '%s' has weight %d",
1476                         comp.getKey().toString(), comp.getValue());
1477             }
1478         }
1479         return compMap;
1480     }
1481 
1482     /**
1483      * Calculate new counts for original values, based on annotation components.
1484      *
1485      * <p>Find the total votes for each component (e.g., "b" in "b|c"). As the "modified" vote for
1486      * the set, use the geometric mean of the components in the set.
1487      *
1488      * <p>Order the sets by that mean value, then by the smallest number of items in the set, then
1489      * the fallback we always use (alphabetical).
1490      *
1491      * @param sortedValues the set of sorted values
1492      * @param voteCount the hash giving the vote count for each value in sortedValues
1493      * @param compMap the hash that maps individual components to cumulative vote counts
1494      *     <p>See http://unicode.org/cldr/trac/ticket/10973
1495      */
calculateNewCountsBasedOnAnnotationComponents( Set<T> sortedValues, HashMap<T, Long> voteCount, HashMap<T, Long> compMap)1496     private void calculateNewCountsBasedOnAnnotationComponents(
1497             Set<T> sortedValues, HashMap<T, Long> voteCount, HashMap<T, Long> compMap) {
1498         voteCount.clear();
1499         annotateTranscript(
1500                 "- Next, the original values get new counts, each based on the geometric mean of the products of all components.");
1501         for (T value : sortedValues) {
1502             List<T> comps = splitAnnotationIntoComponentsList(value);
1503             double product = 1.0;
1504             for (T comp : comps) {
1505                 product *= compMap.get(comp);
1506             }
1507             /* Rounding to long integer here loses precision. We tried multiplying by ten before rounding,
1508              * to reduce problems with different doubles getting rounded to identical longs, but that had
1509              * unfortunate side-effects involving thresholds (see getRequiredVotes). An eventual improvement
1510              * may be to use doubles or floats for all vote counts.
1511              */
1512             Long newCount = Math.round(Math.pow(product, 1.0 / comps.size())); // geometric mean
1513             voteCount.put(value, newCount);
1514             // Don't annotate these here, annotate them once sorted
1515         }
1516     }
1517 
1518     /**
1519      * Split an annotation into a list of components.
1520      *
1521      * <p>For example, split "happy | joyful" into ["happy", "joyful"].
1522      *
1523      * @param value the value like "happy | joyful"
1524      * @return the list like ["happy", "joyful"]
1525      *     <p>Called by makeAnnotationComponentMap and
1526      *     calculateNewCountsBasedOnAnnotationComponents. Short, but needs encapsulation, should be
1527      *     consistent with similar code in DisplayAndInputProcessor.java.
1528      */
splitAnnotationIntoComponentsList(T value)1529     private List<T> splitAnnotationIntoComponentsList(T value) {
1530         return (List<T>) DisplayAndInputProcessor.SPLIT_BAR.splitToList((CharSequence) value);
1531     }
1532 
1533     /**
1534      * Re-sort the set of values to match the adjusted vote counts based on annotation components.
1535      *
1536      * <p>Resolve ties using ULocale.ENGLISH collation for consistency with votesThenUcaCollator.
1537      *
1538      * @param sortedValues the set of sorted values, maybe no longer sorted the way we want
1539      * @param voteCount the hash giving the adjusted vote count for each value in sortedValues
1540      */
resortValuesBasedOnAdjustedVoteCounts( Set<T> sortedValues, HashMap<T, Long> voteCount)1541     private void resortValuesBasedOnAdjustedVoteCounts(
1542             Set<T> sortedValues, HashMap<T, Long> voteCount) {
1543         List<T> list = new ArrayList<>(sortedValues);
1544         list.sort(
1545                 (v1, v2) -> {
1546                     long c1 = voteCount.get(v1), c2 = voteCount.get(v2);
1547                     if (c1 != c2) {
1548                         return (c1 < c2) ? 1 : -1; // decreasing numeric order (most votes wins)
1549                     }
1550                     int size1 = splitAnnotationIntoComponentsList(v1).size();
1551                     int size2 = splitAnnotationIntoComponentsList(v2).size();
1552                     if (size1 != size2) {
1553                         return (size1 < size2)
1554                                 ? -1
1555                                 : 1; // increasing order of size (smallest set wins)
1556                     }
1557                     return englishCollator.compare(String.valueOf(v1), String.valueOf(v2));
1558                 });
1559         sortedValues.clear();
1560         sortedValues.addAll(list);
1561     }
1562 
1563     /**
1564      * For annotation votes, if the set that so far is winning has one or more supersets with
1565      * "superior" (see below) raw vote count, promote those supersets to become the new winner, and
1566      * also the new second place if there are two or more superior supersets.
1567      *
1568      * <p>That is, after finding the set X with the largest geometric mean, check whether there are
1569      * any supersets with "superior" raw votes, and that don't exceed the width limit. If so,
1570      * promote Y, the one of those supersets with the most raw votes (using the normal tie breaker),
1571      * to be the winning set.
1572      *
1573      * <p>"Superior" here means that rawVote(Y) ≥ rawVote(X) + 2, where the value 2 (see
1574      * requiredGap) is for the purpose of requiring at least one non-guest vote.
1575      *
1576      * <p>If any other "superior" supersets exist, promote to second place the one with the next
1577      * most raw votes.
1578      *
1579      * <p>Accomplish promotion by increasing vote counts in the voteCount hash.
1580      *
1581      * @param sortedValues the set of sorted values
1582      * @param voteCount the vote count for each value in sortedValues AFTER
1583      *     calculateNewCountsBasedOnAnnotationComponents; it gets modified if superior subsets exist
1584      * @param rawVoteCount the vote count for each value in sortedValues BEFORE
1585      *     calculateNewCountsBasedOnAnnotationComponents; rawVoteCount is not changed by this
1586      *     function
1587      *     <p>Reference: https://unicode.org/cldr/trac/ticket/10973
1588      */
promoteSuperiorAnnotationSuperset( Set<T> sortedValues, HashMap<T, Long> voteCount, HashMap<T, Long> rawVoteCount)1589     private void promoteSuperiorAnnotationSuperset(
1590             Set<T> sortedValues, HashMap<T, Long> voteCount, HashMap<T, Long> rawVoteCount) {
1591         final long requiredGap = 2;
1592         T oldWinner = null;
1593         long oldWinnerRawCount = 0;
1594         LinkedHashSet<T> oldWinnerComps = null;
1595         LinkedHashSet<T> superiorSupersets = null;
1596         for (T value : sortedValues) {
1597             // Annotate the means here
1598             final long rawCount = rawVoteCount.get(value);
1599             final long newCount = voteCount.get(value);
1600             if (rawCount != newCount) {
1601                 annotateTranscript("-- Value '%s' has updated value '%d'", value, newCount);
1602             }
1603             if (oldWinner == null) {
1604                 oldWinner = value;
1605                 oldWinnerRawCount = rawVoteCount.get(value);
1606                 oldWinnerComps = new LinkedHashSet<>(splitAnnotationIntoComponentsList(value));
1607             } else {
1608                 Set<T> comps = new LinkedHashSet<>(splitAnnotationIntoComponentsList(value));
1609                 if (comps.size() <= CheckWidths.MAX_COMPONENTS_PER_ANNOTATION
1610                         && comps.containsAll(oldWinnerComps)
1611                         && rawVoteCount.get(value) >= oldWinnerRawCount + requiredGap) {
1612                     if (superiorSupersets == null) {
1613                         superiorSupersets = new LinkedHashSet<>();
1614                     }
1615                     superiorSupersets.add(value);
1616                 }
1617             }
1618         }
1619         if (superiorSupersets != null) {
1620             // Sort the supersets by raw vote count, then make their adjusted vote counts higher
1621             // than the old winner's.
1622             resortValuesBasedOnAdjustedVoteCounts(superiorSupersets, rawVoteCount);
1623             T newWinner = null, newSecond; // only adjust votes for first and second place
1624             for (T value : superiorSupersets) {
1625                 if (newWinner == null) {
1626                     newWinner = value;
1627                     long newWinnerCount = voteCount.get(oldWinner) + 2;
1628                     annotateTranscript(
1629                             "- Optimal value (O) '%s' was promoted to value '%d' due to having a superior raw vote count",
1630                             newWinner, newWinnerCount);
1631                     voteCount.put(newWinner, newWinnerCount); // more than oldWinner and newSecond
1632                 } else {
1633                     newSecond = value;
1634                     long newSecondCount = voteCount.get(oldWinner) + 1;
1635                     annotateTranscript(
1636                             "- Next value (N) '%s' was promoted to value '%d' due to having a superior raw vote count",
1637                             newSecond, newSecondCount);
1638                     voteCount.put(
1639                             newSecond, newSecondCount); // more than oldWinner, less than newWinner
1640                     break;
1641                 }
1642             }
1643             resortValuesBasedOnAdjustedVoteCounts(sortedValues, voteCount);
1644         }
1645     }
1646 
1647     /**
1648      * Given a nonempty list of sorted values, and a hash with their vote counts, set these members
1649      * of this VoteResolver: winningValue, nValue, valuesWithSameVotes (which is empty when this
1650      * function is called).
1651      *
1652      * @param sortedValues the set of sorted values
1653      * @param voteCount the hash giving the vote count for each value
1654      * @return an array of two longs, the weights for the best and next-best values.
1655      */
setBestNextAndSameVoteValues(Set<T> sortedValues, HashMap<T, Long> voteCount)1656     private long[] setBestNextAndSameVoteValues(Set<T> sortedValues, HashMap<T, Long> voteCount) {
1657 
1658         long[] weightArray = new long[2];
1659         nValue = null;
1660 
1661         /*
1662          * Loop through the sorted values, at least the first (best) for winningValue,
1663          * and the second (if any) for nValue (else nValue stays null),
1664          * and subsequent values that have as many votes as the first,
1665          * to add to valuesWithSameVotes.
1666          */
1667         int i = -1;
1668         Iterator<T> iterator = sortedValues.iterator();
1669         for (T value : sortedValues) {
1670             ++i;
1671             long valueWeight = voteCount.get(value);
1672             if (i == 0) {
1673                 setWinningValue(value);
1674                 weightArray[0] = valueWeight;
1675                 valuesWithSameVotes.add(value);
1676                 annotateTranscript(
1677                         "The optimal value (O) is '%s', with a weight of %d",
1678                         winningValue, valueWeight);
1679                 if (sortedValues.size() == 1) {
1680                     annotateTranscript("- No other values received votes."); // uncontested
1681                 }
1682             } else {
1683                 if (i == 1) {
1684                     // get the next item if there is one
1685                     if (iterator.hasNext()) {
1686                         nValue = value;
1687                         weightArray[1] = valueWeight;
1688                         annotateNextBestValue(weightArray[0], weightArray[1], winningValue, nValue);
1689                     }
1690                 }
1691                 if (valueWeight == weightArray[0]) {
1692                     valuesWithSameVotes.add(value);
1693                 } else {
1694                     break;
1695                 }
1696             }
1697         }
1698         return weightArray;
1699     }
1700 
1701     /**
1702      * Compute the status for the winning value. See: https://cldr.unicode.org/index/process
1703      *
1704      * @param O the weight (vote count) for the best value
1705      * @param N the weight (vote count) for the next-best value
1706      * @return the Status
1707      */
computeStatus(long O, long N)1708     private Status computeStatus(long O, long N) {
1709         if (O > N) {
1710             final int requiredVotes = getRequiredVotes();
1711             if (O >= requiredVotes) {
1712                 final Status computedStatus = Status.approved;
1713                 annotateTranscript("O>N, and O>%d: %s", requiredVotes, computedStatus);
1714                 return computedStatus;
1715             }
1716             if (O >= 4 && Status.contributed.compareTo(baselineStatus) > 0) {
1717                 final Status computedStatus = Status.contributed;
1718                 annotateTranscript(
1719                         "O>=4, and oldstatus (%s)<contributed: %s", baselineStatus, computedStatus);
1720                 return computedStatus;
1721             }
1722             if (O >= 2) {
1723                 final int G = organizationToValueAndVote.getOrgCount(winningValue);
1724                 if (G >= 2) {
1725                     final Status computedStatus = Status.contributed;
1726                     annotateTranscript("O>=2, and G (%d)>=2: %s", G, computedStatus);
1727                     return computedStatus;
1728                 }
1729             }
1730         }
1731         if (O >= N) {
1732             if (O >= 2) {
1733                 final Status computedStatus = Status.provisional;
1734                 annotateTranscript("O>=N and O>=2: %s", computedStatus);
1735                 return computedStatus;
1736             }
1737         }
1738 
1739         // otherwise: unconfirmed
1740         final Status computedStatus = Status.unconfirmed;
1741         annotateTranscript("O was not high enough: %s", computedStatus);
1742         return computedStatus;
1743     }
1744 
getPossibleWinningStatus()1745     private Status getPossibleWinningStatus() {
1746         if (!resolved) {
1747             resolveVotes();
1748         }
1749         Status possibleStatus = computeStatus(organizationToValueAndVote.getBestPossibleVote(), 0);
1750         return possibleStatus.compareTo(winningStatus) > 0 ? possibleStatus : winningStatus;
1751     }
1752 
1753     /**
1754      * If the winning item is not approved, and if all the people who voted had voted for the
1755      * winning item, would it have made contributed or approved?
1756      *
1757      * @return
1758      */
isDisputed()1759     public boolean isDisputed() {
1760         if (!resolved) {
1761             resolveVotes();
1762         }
1763         if (winningStatus.compareTo(VoteResolver.Status.contributed) >= 0) {
1764             return false;
1765         }
1766         VoteResolver.Status possibleStatus = getPossibleWinningStatus();
1767         return possibleStatus.compareTo(Status.contributed) >= 0;
1768     }
1769 
getWinningStatus()1770     public Status getWinningStatus() {
1771         if (!resolved) {
1772             resolveVotes();
1773         }
1774         return winningStatus;
1775     }
1776 
1777     /**
1778      * Returns O Value as described in http://cldr.unicode.org/index/process#TOC-Voting-Process. Not
1779      * always the same as the Winning Value.
1780      *
1781      * @return
1782      */
getOValue()1783     private T getOValue() {
1784         if (!resolved) {
1785             resolveVotes();
1786         }
1787         return oValue;
1788     }
1789 
1790     /**
1791      * Returns N Value as described in http://cldr.unicode.org/index/process#TOC-Voting-Process. Not
1792      * always the same as the Winning Value.
1793      *
1794      * @return
1795      */
getNValue()1796     private T getNValue() {
1797         if (!resolved) {
1798             resolveVotes();
1799         }
1800         return nValue;
1801     }
1802 
1803     /**
1804      * Returns Winning Value as described in
1805      * http://cldr.unicode.org/index/process#TOC-Voting-Process. Not always the same as the O Value.
1806      *
1807      * @return
1808      */
getWinningValue()1809     public T getWinningValue() {
1810         if (!resolved) {
1811             resolveVotes();
1812         }
1813         return winningValue;
1814     }
1815 
1816     /**
1817      * Set the Winning Value; if the given value matches Bailey, change it to INHERITANCE_MARKER
1818      *
1819      * @param value the value to set (prior to changeBaileyToInheritance)
1820      */
setWinningValue(T value)1821     private void setWinningValue(T value) {
1822         if (DROP_HARD_INHERITANCE) {
1823             winningValue = changeBaileyToInheritance(value);
1824         } else {
1825             winningValue = value;
1826         }
1827     }
1828 
getValuesWithSameVotes()1829     public List<T> getValuesWithSameVotes() {
1830         if (!resolved) {
1831             resolveVotes();
1832         }
1833         return new ArrayList<>(valuesWithSameVotes);
1834     }
1835 
getConflictedOrganizations()1836     public EnumSet<Organization> getConflictedOrganizations() {
1837         if (!resolved) {
1838             resolveVotes();
1839         }
1840         return conflictedOrganizations;
1841     }
1842 
1843     /**
1844      * What value did this organization vote for?
1845      *
1846      * @param org
1847      * @return
1848      */
getOrgVote(Organization org)1849     public T getOrgVote(Organization org) {
1850         return organizationToValueAndVote.getOrgVote(org);
1851     }
1852 
getOrgToVotes(Organization org)1853     public Map<T, Long> getOrgToVotes(Organization org) {
1854         return organizationToValueAndVote.getOrgToVotes(org);
1855     }
1856 
getNameTime()1857     public Map<String, Long> getNameTime() {
1858         return organizationToValueAndVote.getNameTime();
1859     }
1860 
1861     /**
1862      * Get a String representation of this VoteResolver. This is sent to the client as
1863      * "voteResolver.raw" and is used only for debugging.
1864      *
1865      * <p>Compare SurveyAjax.JSONWriter.wrap(VoteResolver<String>) which creates the data actually
1866      * used by the client.
1867      */
1868     @Override
toString()1869     public String toString() {
1870         return "{"
1871                 + "bailey: "
1872                 + (organizationToValueAndVote.baileySet
1873                         ? ("“" + organizationToValueAndVote.baileyValue + "” ")
1874                         : "none ")
1875                 + "baseline: {"
1876                 + baselineValue
1877                 + ", "
1878                 + baselineStatus
1879                 + "}, "
1880                 + organizationToValueAndVote
1881                 + ", sameVotes: "
1882                 + valuesWithSameVotes
1883                 + ", O: "
1884                 + getOValue()
1885                 + ", N: "
1886                 + getNValue()
1887                 + ", totals: "
1888                 + totals
1889                 + ", winning: {"
1890                 + getWinningValue()
1891                 + ", "
1892                 + getWinningStatus()
1893                 + "}"
1894                 + "}";
1895     }
1896 
getIdToPath(String fileName)1897     public static Map<Integer, String> getIdToPath(String fileName) {
1898         XPathTableHandler myHandler = new XPathTableHandler();
1899         XMLFileReader xfr = new XMLFileReader().setHandler(myHandler);
1900         xfr.read(fileName, XMLFileReader.CONTENT_HANDLER | XMLFileReader.ERROR_HANDLER, false);
1901         return myHandler.pathIdToPath;
1902     }
1903 
1904     static class XPathTableHandler extends XMLFileReader.SimpleHandler {
1905         Matcher matcher = Pattern.compile("id=\"([0-9]+)\"").matcher("");
1906         Map<Integer, String> pathIdToPath = new HashMap<>();
1907 
1908         @Override
handlePathValue(String path, String value)1909         public void handlePathValue(String path, String value) {
1910             // <xpathTable host="tintin.local" date="Tue Apr 29 14:34:32 PDT 2008" count="18266" >
1911             // <xpath
1912             // id="1">//ldml/dates/calendars/calendar[@type="gregorian"]/dateFormats/dateFormatLength[@type="short"]/dateFormat[@type="standard"]/pattern[@type="standard"]</xpath>
1913             if (!matcher.reset(path).find()) {
1914                 throw new IllegalArgumentException("Unknown path " + path);
1915             }
1916             pathIdToPath.put(Integer.parseInt(matcher.group(1)), value);
1917         }
1918     }
1919 
getBaseToAlternateToInfo( String fileName, VoterInfoList vil)1920     public static Map<Integer, Map<Integer, CandidateInfo>> getBaseToAlternateToInfo(
1921             String fileName, VoterInfoList vil) {
1922         try {
1923             VotesHandler myHandler = new VotesHandler(vil);
1924             XMLFileReader xfr = new XMLFileReader().setHandler(myHandler);
1925             xfr.read(fileName, XMLFileReader.CONTENT_HANDLER | XMLFileReader.ERROR_HANDLER, false);
1926             return myHandler.basepathToInfo;
1927         } catch (Exception e) {
1928             throw new IllegalArgumentException("Can't handle file: " + fileName, e);
1929         }
1930     }
1931 
1932     public enum Type {
1933         proposal,
1934         optimal
1935     }
1936 
1937     public static class CandidateInfo {
1938         public Status oldStatus;
1939         public Type surveyType;
1940         public Status surveyStatus;
1941         public Set<Integer> voters = new TreeSet<>();
1942         private final VoterInfoList voterInfoList;
1943 
CandidateInfo(VoterInfoList vil)1944         CandidateInfo(VoterInfoList vil) {
1945             this.voterInfoList = vil;
1946         }
1947 
1948         @Override
toString()1949         public String toString() {
1950             StringBuilder voterString = new StringBuilder("{");
1951             for (int voter : voters) {
1952                 VoterInfo voterInfo = voterInfoList.get(voter);
1953                 if (voterString.length() > 1) {
1954                     voterString.append(" ");
1955                 }
1956                 voterString.append(voter);
1957                 if (voterInfo != null) {
1958                     voterString.append(" ").append(voterInfo);
1959                 }
1960             }
1961             voterString.append("}");
1962             return "{oldStatus: "
1963                     + oldStatus
1964                     + ", surveyType: "
1965                     + surveyType
1966                     + ", surveyStatus: "
1967                     + surveyStatus
1968                     + ", voters: "
1969                     + voterString
1970                     + "};";
1971         }
1972     }
1973 
1974     /*
1975      * <locale-votes host="tintin.local" date="Tue Apr 29 14:34:32 PDT 2008"
1976      * oldVersion="1.5.1" currentVersion="1.6" resolved="false" locale="zu">
1977      * <row baseXpath="1">
1978      * <item xpath="2855" type="proposal" id="1" status="unconfirmed">
1979      * <old status="unconfirmed"/>
1980      * </item>
1981      * <item xpath="1" type="optimal" id="56810" status="confirmed">
1982      * <vote user="210"/>
1983      * </item>
1984      * </row>
1985      * ...
1986      * A base path has a set of candidates. Each candidate has various items of information.
1987      */
1988     static class VotesHandler extends XMLFileReader.SimpleHandler {
1989         private final VoterInfoList voterInfoList;
1990 
VotesHandler(VoterInfoList vil)1991         VotesHandler(VoterInfoList vil) {
1992             this.voterInfoList = vil;
1993         }
1994 
1995         Map<Integer, Map<Integer, CandidateInfo>> basepathToInfo = new TreeMap<>();
1996 
1997         @Override
handlePathValue(String path, String value)1998         public void handlePathValue(String path, String value) {
1999             try {
2000                 XPathParts parts = XPathParts.getFrozenInstance(path);
2001                 if (parts.size() < 2) {
2002                     // empty data
2003                     return;
2004                 }
2005                 int baseId = Integer.parseInt(parts.getAttributeValue(1, "baseXpath"));
2006                 Map<Integer, CandidateInfo> info =
2007                         basepathToInfo.computeIfAbsent(baseId, k -> new TreeMap<>());
2008                 int itemId = Integer.parseInt(parts.getAttributeValue(2, "xpath"));
2009                 CandidateInfo candidateInfo = info.get(itemId);
2010                 if (candidateInfo == null) {
2011                     info.put(itemId, candidateInfo = new CandidateInfo(voterInfoList));
2012                     candidateInfo.surveyType = Type.valueOf(parts.getAttributeValue(2, "type"));
2013                     candidateInfo.surveyStatus =
2014                             Status.valueOf(
2015                                     fixBogusDraftStatusValues(
2016                                             parts.getAttributeValue(2, "status")));
2017                     // ignore id
2018                 }
2019                 if (parts.size() < 4) {
2020                     return;
2021                 }
2022                 final String lastElement = parts.getElement(3);
2023                 if (lastElement.equals("old")) {
2024                     candidateInfo.oldStatus =
2025                             Status.valueOf(
2026                                     fixBogusDraftStatusValues(
2027                                             parts.getAttributeValue(3, "status")));
2028                 } else if (lastElement.equals("vote")) {
2029                     candidateInfo.voters.add(Integer.parseInt(parts.getAttributeValue(3, "user")));
2030                 } else {
2031                     throw new IllegalArgumentException("unknown option: " + path);
2032                 }
2033             } catch (Exception e) {
2034                 throw new IllegalArgumentException("Can't handle path: " + path, e);
2035             }
2036         }
2037     }
2038 
2039     public static class UnknownVoterException extends RuntimeException {
2040         private static final long serialVersionUID = 3430877787936678609L;
2041         int voter;
2042 
UnknownVoterException(int voter)2043         public UnknownVoterException(int voter) {
2044             this.voter = voter;
2045         }
2046 
2047         @Override
toString()2048         public String toString() {
2049             return "Unknown voter: " + voter;
2050         }
2051     }
2052 
fixBogusDraftStatusValues(String attributeValue)2053     private static String fixBogusDraftStatusValues(String attributeValue) {
2054         if (attributeValue == null) return "approved";
2055         if ("confirmed".equals(attributeValue)) return "approved";
2056         if ("true".equals(attributeValue)) return "unconfirmed";
2057         if ("unknown".equals(attributeValue)) return "unconfirmed";
2058         return attributeValue;
2059     }
2060 
2061     /*
2062      * TODO: either delete this or explain why it's needed
2063      */
size()2064     public int size() {
2065         return values.size();
2066     }
2067 
2068     /**
2069      * Returns a map from value to resolved vote count, in descending order. If the winning item is
2070      * not there, insert at the front. If the baseline (trunk) item is not there, insert at the end.
2071      *
2072      * <p>This map includes intra-org disputes.
2073      *
2074      * @return the map
2075      */
getResolvedVoteCountsIncludingIntraOrgDisputes()2076     public Map<T, Long> getResolvedVoteCountsIncludingIntraOrgDisputes() {
2077         if (!resolved) {
2078             resolveVotes();
2079         }
2080         Map<T, Long> result = new LinkedHashMap<>();
2081         if (winningValue != null && !totals.containsKey(winningValue)) {
2082             result.put(winningValue, 0L);
2083         }
2084         for (T value : totals.getKeysetSortedByCount(false, votesThenUcaCollator)) {
2085             result.put(value, totals.get(value));
2086         }
2087         if (baselineValue != null && !totals.containsKey(baselineValue)) {
2088             result.put(baselineValue, 0L);
2089         }
2090         for (T value :
2091                 organizationToValueAndVote.allVotesIncludingIntraOrgDispute.getMap().keySet()) {
2092             if (!result.containsKey(value)) {
2093                 result.put(value, 0L);
2094             }
2095         }
2096         if (DEBUG) {
2097             System.out.println("getResolvedVoteCountsIncludingIntraOrgDisputes :" + result);
2098         }
2099         return result;
2100     }
2101 
getStatusForOrganization(Organization orgOfUser)2102     public VoteStatus getStatusForOrganization(Organization orgOfUser) {
2103         if (!resolved) {
2104             resolveVotes();
2105         }
2106         if (Status.provisional.compareTo(winningStatus) >= 0) {
2107             // If the value is provisional, it needs more votes.
2108             return VoteStatus.provisionalOrWorse;
2109         }
2110         T orgVote = organizationToValueAndVote.getOrgVoteRaw(orgOfUser);
2111         if (!equalsOrgVote(winningValue, orgVote)) {
2112             // We voted and lost
2113             return VoteStatus.losing;
2114         }
2115         final int itemsWithVotes =
2116                 DROP_HARD_INHERITANCE ? totals.size() : countDistinctValuesWithVotes();
2117         if (itemsWithVotes > 1) {
2118             // If there are votes for two "distinct" items, we should look at them.
2119             return VoteStatus.disputed;
2120         }
2121         final T singleVotedItem = getSingleVotedItem();
2122         if (!equalsOrgVote(winningValue, singleVotedItem)) {
2123             // If someone voted but didn't win
2124             return VoteStatus.disputed;
2125         }
2126         if (itemsWithVotes == 0) {
2127             // The value is ok, but we capture that there are no votes, for revealing items like
2128             // unsync'ed
2129             return VoteStatus.ok_novotes;
2130         } else {
2131             // We voted, we won, value is approved, no disputes, have votes
2132             return VoteStatus.ok;
2133         }
2134     }
2135 
2136     /**
2137      * Returns value of voted item, in case there is exactly 1.
2138      *
2139      * @return
2140      */
getSingleVotedItem()2141     private T getSingleVotedItem() {
2142         return totals.size() != 1 ? null : totals.iterator().next();
2143     }
2144 
2145     /**
2146      * Should these two values be treated as equivalent for getStatusForOrganization?
2147      *
2148      * @param value
2149      * @param orgVote
2150      * @return true if they are equivalent, false if they are distinct
2151      */
equalsOrgVote(T value, T orgVote)2152     private boolean equalsOrgVote(T value, T orgVote) {
2153         return orgVote == null
2154                 || orgVote.equals(value)
2155                 || (CldrUtility.INHERITANCE_MARKER.equals(value)
2156                         && orgVote.equals(organizationToValueAndVote.baileyValue))
2157                 || (CldrUtility.INHERITANCE_MARKER.equals(orgVote)
2158                         && value.equals(organizationToValueAndVote.baileyValue));
2159     }
2160 
2161     /**
2162      * Count the distinct values that have votes.
2163      *
2164      * <p>For this purpose, if there are both votes for inheritance and votes for the specific value
2165      * matching the inherited (bailey) value, they are not "distinct": count them as a single value.
2166      *
2167      * @return the number of distinct values
2168      */
countDistinctValuesWithVotes()2169     private int countDistinctValuesWithVotes() {
2170         if (!resolved) { // must be resolved for bothInheritanceAndBaileyHadVotes
2171             throw new RuntimeException("countDistinctValuesWithVotes !resolved");
2172         }
2173         int count = organizationToValueAndVote.allVotesIncludingIntraOrgDispute.size();
2174         if (count > 1 && bothInheritanceAndBaileyHadVotes) {
2175             return count - 1; // prevent showing as "disputed" in dashboard
2176         }
2177         return count;
2178     }
2179 
2180     /**
2181      * Should this VoteResolver use keyword annotation voting?
2182      *
2183      * <p>Apply special voting method adjustAnnotationVoteCounts only to certain keyword annotations
2184      * that can have bar-separated values like "happy | joyful".
2185      *
2186      * <p>The paths for keyword annotations start with "//ldml/annotations/annotation" and do NOT
2187      * include Emoji.TYPE_TTS. Both name paths (cf. namePath, getNamePaths) and keyword paths (cf.
2188      * keywordPath, getKeywordPaths) have "//ldml/annotations/annotation". Name paths include
2189      * Emoji.TYPE_TTS, and keyword paths don't. Special voting is only for keyword paths, not for
2190      * name paths. Compare path dependencies in DisplayAndInputProcessor.java. See also
2191      * VoteResolver.splitAnnotationIntoComponentsList.
2192      *
2193      * @return true or false
2194      */
isUsingKeywordAnnotationVoting()2195     private boolean isUsingKeywordAnnotationVoting() {
2196         if (pathHeader == null) {
2197             return false; // this happens in some tests
2198         }
2199         final String path = pathHeader.getOriginalPath();
2200         return AnnotationUtil.pathIsAnnotation(path) && !path.contains(Emoji.TYPE_TTS);
2201     }
2202 
2203     /**
2204      * Is the value locked for this locale+path?
2205      *
2206      * @return true or false
2207      */
isValueLocked()2208     public boolean isValueLocked() {
2209         return valueIsLocked;
2210     }
2211 
2212     /**
2213      * Can a user who makes a losing vote flag the locale+path? I.e., is the locale+path locked
2214      * and/or does it require HIGH_BAR votes?
2215      *
2216      * @return true or false
2217      */
canFlagOnLosing()2218     public boolean canFlagOnLosing() {
2219         return valueIsLocked || (getRequiredVotes() == HIGH_BAR);
2220     }
2221 
2222     /**
2223      * Calculate VoteResolver.Status
2224      *
2225      * @param baselineFile the 'baseline' file to use
2226      * @param path path the xpath
2227      * @return the Status
2228      */
calculateStatus(CLDRFile baselineFile, String path)2229     public static Status calculateStatus(CLDRFile baselineFile, String path) {
2230         String fullXPath = baselineFile.getFullXPath(path);
2231         if (fullXPath == null) {
2232             fullXPath = path;
2233         }
2234         final XPathParts xpp = XPathParts.getFrozenInstance(fullXPath);
2235         final String draft = xpp.getAttributeValue(-1, LDMLConstants.DRAFT);
2236         Status status = draft == null ? Status.approved : VoteResolver.Status.fromString(draft);
2237 
2238         /*
2239          * Reset to missing if the value is inherited from root or code-fallback, unless the XML actually
2240          * contains INHERITANCE_MARKER. Pass false for skipInheritanceMarker so that status will not be
2241          * missing for explicit INHERITANCE_MARKER.
2242          */
2243         final String srcid =
2244                 baselineFile.getSourceLocaleIdExtended(
2245                         path, null, false /* skipInheritanceMarker */);
2246         if (srcid.equals(XMLSource.CODE_FALLBACK_ID)) {
2247             status = Status.missing;
2248         } else if (srcid.equals("root")) {
2249             if (!srcid.equals(baselineFile.getLocaleID())) {
2250                 status = Status.missing;
2251             }
2252         }
2253         return status;
2254     }
2255     /**
2256      * Get the possibly modified value. If value matches the bailey value or inheritance marker,
2257      * possibly change it from bailey value to inheritance marker, or vice-versa, as needed to meet
2258      * these requirements: 1. If the path changes when getting bailey, then we are inheriting
2259      * sideways. We need to use a hard value. 2. If the value is different from the bailey value,
2260      * can't use inheritance; we need a hard value. 3. Otherwise we use inheritance marker.
2261      *
2262      * <p>These requirements are pragmatic, to work around limitations of the current inheritance
2263      * algorithm, which is hyper-sensitive to the distinction between inheritance marker and bailey,
2264      * which, depending on that distinction, unintentionally tends to change lateral inheritance to
2265      * vertical inheritance, or vice-versa.
2266      *
2267      * <p>This method has consequences affecting vote resolution. For example, assume
2268      * DROP_HARD_INHERITANCE is true. If a user votes for what is currently the inherited value, and
2269      * these requirements call for using inheritance marker, then their vote is stored as
2270      * inheritance marker in the db; if the parent value then changes (even during same release
2271      * cycle), the vote is still a vote for inheritance -- that is how soft inheritence has long
2272      * been intended to work. In the cases where this method returns the hard value matching bailey,
2273      * the user's vote is stored in the db as that hard value; if the parent value then changes, the
2274      * user's vote does not change -- this differs from what we'd like ideally (which is for all
2275      * inh. votes to be "soft"). If and when the inheritance algorithm changes to reduce or
2276      * eliminate the problematic aspects of the hard/soft distinction, this method might no longer
2277      * be needed.
2278      *
2279      * <p>Reference: https://unicode-org.atlassian.net/browse/CLDR-16560
2280      *
2281      * @param path the path
2282      * @param value the input value
2283      * @param cldrFile the CLDRFile for determining inheritance
2284      * @return the possibly modified value
2285      */
reviseInheritanceAsNeeded(String path, String value, CLDRFile cldrFile)2286     public static String reviseInheritanceAsNeeded(String path, String value, CLDRFile cldrFile) {
2287         if (!DROP_HARD_INHERITANCE) {
2288             return value;
2289         }
2290         if (!cldrFile.isResolved()) {
2291             throw new InternalCldrException("must be resolved");
2292         }
2293         Output<String> pathWhereFound = new Output<>();
2294         Output<String> localeWhereFound = new Output<>();
2295         String baileyValue = cldrFile.getBaileyValue(path, pathWhereFound, localeWhereFound);
2296         if (baileyValue != null
2297                 && (CldrUtility.INHERITANCE_MARKER.equals(value) || baileyValue.equals(value))) {
2298             // TODO: decide whether to continue treating GlossonymConstructor.PSEUDO_PATH
2299             // (constructed values) as lateral inheritance. This method originally did not
2300             // take constructed values into account, so it implicitly treated constructed
2301             // values as laterally inherited, given that pathWhereFound doesn't equal path.
2302             // This original behavior corresponds to CONSTRUCTED_PSEUDO_PATH_NOT_LATERAL = false.
2303             // Reference: https://unicode-org.atlassian.net/browse/CLDR-16372
2304             final boolean CONSTRUCTED_PSEUDO_PATH_NOT_LATERAL = false;
2305             value =
2306                     (pathWhereFound.value.equals(path)
2307                                     || (CONSTRUCTED_PSEUDO_PATH_NOT_LATERAL
2308                                             && GlossonymConstructor.PSEUDO_PATH.equals(
2309                                                     pathWhereFound.value)))
2310                             ? CldrUtility.INHERITANCE_MARKER
2311                             : baileyValue;
2312         }
2313         return value;
2314     }
2315 }
2316