Skip to content

Commit 9ab9b88

Browse files
pmachapmanpmachapman
committed
Port transductive alignment model
1 parent dac2d89 commit 9ab9b88

10 files changed

Lines changed: 427 additions & 14 deletions

File tree

‎src/SIL.Machine.Translation.Thot/CorporaExtensions.cs‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Threading.Tasks;
5+
using SIL.Machine.Corpora;
6+
using SIL.Machine.Utils;
7+
8+
namespace SIL.Machine.Translation.Thot
9+
{
10+
public static class IParallelTextCorpusExtensions
11+
{
12+
public static async Task<IParallelTextCorpus> WordAlignAsync(
13+
this IParallelTextCorpus corpus,
14+
ThotWordAlignmentModelType modelType,
15+
int batchSize = 1024,
16+
SymmetrizationHeuristic symmetrizationHeuristic = SymmetrizationHeuristic.GrowDiagFinalAnd,
17+
IProgress<ProgressStatus> progress = null
18+
)
19+
{
20+
// TODO: Warning! This model must be disposed correctly.
21+
var model = new ThotSymmetrizedWordAlignmentModel(
22+
ThotWordAlignmentModel.Create(modelType),
23+
ThotWordAlignmentModel.Create(modelType)
24+
)
25+
{
26+
Heuristic = symmetrizationHeuristic,
27+
// Retain the alignments computed during training so that the corpus can be aligned
28+
// without a separate, potentially expensive, inference pass.
29+
EmitTrainingAlignments = true,
30+
};
31+
32+
using (ITrainer trainer = model.CreateTrainer(corpus))
33+
{
34+
await trainer.TrainAsync(progress);
35+
await trainer.SaveAsync();
36+
}
37+
38+
return corpus.WordAlign(model, batchSize);
39+
}
40+
41+
public static IParallelTextCorpus WordAlign(
42+
this IParallelTextCorpus corpus,
43+
ThotSymmetrizedWordAlignmentModel model,
44+
int batchSize = 1024
45+
)
46+
{
47+
if (model.EmitTrainingAlignments)
48+
return new TransductiveWordAlignParallelTextCorpus(corpus, model);
49+
50+
return CorporaExtensions.WordAlign(corpus, model, batchSize);
51+
}
52+
53+
private class TransductiveWordAlignParallelTextCorpus : ParallelTextCorpusBase
54+
{
55+
private readonly IParallelTextCorpus _corpus;
56+
private readonly ITransductiveWordAlignmentModel _model;
57+
58+
public TransductiveWordAlignParallelTextCorpus(
59+
IParallelTextCorpus corpus,
60+
ITransductiveWordAlignmentModel model
61+
)
62+
{
63+
_corpus = corpus;
64+
_model = model;
65+
}
66+
67+
public override bool IsSourceTokenized => _corpus.IsSourceTokenized;
68+
69+
public override bool IsTargetTokenized => _corpus.IsTargetTokenized;
70+
71+
public override IEnumerable<ParallelTextRow> GetRows(IEnumerable<string> textIds)
72+
{
73+
// The training alignments are keyed by the order in which the sentence pairs were added
74+
// during training, so the full corpus must be iterated to keep the index in sync; rows that
75+
// are not in the requested texts are skipped rather than filtered out of the enumeration.
76+
var textIdList = textIds?.ToList();
77+
List<ParallelTextRow> rows = _corpus.GetRows().ToList();
78+
for (int i = 0; i < rows.Count; i++)
79+
{
80+
ParallelTextRow row = rows[i];
81+
if (textIdList != null && !textIdList.Contains(row.TextId))
82+
continue;
83+
84+
WordAlignmentMatrix alignment = _model.GetTrainingAlignment(i);
85+
WordAlignmentMatrix knownAlignment = row.CreateAlignmentMatrix();
86+
if (knownAlignment != null)
87+
{
88+
knownAlignment.PrioritySymmetrizeWith(alignment);
89+
alignment = knownAlignment;
90+
}
91+
92+
IReadOnlyCollection<AlignedWordPair> wordPairs = alignment.ToAlignedWordPairs();
93+
if (_model is IWordAlignmentModel wordAlignmentModel)
94+
{
95+
wordAlignmentModel.ComputeAlignedWordPairScores(
96+
row.SourceSegment,
97+
row.TargetSegment,
98+
wordPairs
99+
);
100+
}
101+
102+
row.AlignedWordPairs = wordPairs;
103+
yield return row;
104+
}
105+
}
106+
}
107+
}
108+
}

‎src/SIL.Machine.Translation.Thot/SIL.Machine.Translation.Thot.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
<Import Project="../AssemblyInfo.props" />
1313

1414
<ItemGroup>
15-
<PackageReference Include="Thot" Version="3.5.0" />
15+
<PackageReference Include="Thot" Version="3.5.1" />
1616
</ItemGroup>
1717

1818
<ItemGroup>

‎src/SIL.Machine.Translation.Thot/Thot.cs‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,21 @@ uint capacity
172172
[DllImport("thot", CallingConvention = CallingConvention.Cdecl)]
173173
public static extern uint swAlignModel_getMaxSentenceLength(IntPtr swAlignModelHandle);
174174

175+
[DllImport("thot", CallingConvention = CallingConvention.Cdecl)]
176+
public static extern uint swAlignModel_getNumSentencePairs(IntPtr swAlignModelHandle);
177+
178+
[DllImport("thot", CallingConvention = CallingConvention.Cdecl)]
179+
public static extern double swAlignModel_getTrainingAlignment(
180+
IntPtr swAlignModelHandle,
181+
uint n,
182+
IntPtr matrix,
183+
ref uint iLen,
184+
ref uint jLen
185+
);
186+
187+
[DllImport("thot", CallingConvention = CallingConvention.Cdecl)]
188+
public static extern void swAlignModel_setEmitTrainingAlignments(IntPtr swAlignModelHandle, bool value);
189+
175190
[DllImport("thot", CallingConvention = CallingConvention.Cdecl)]
176191
public static extern void swAlignModel_setVariationalBayes(IntPtr swAlignModelHandle, bool variationalBayes);
177192

‎src/SIL.Machine.Translation.Thot/ThotSymmetrizedWordAlignmentModel.cs‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
namespace SIL.Machine.Translation.Thot
2+
{
3+
public class ThotSymmetrizedWordAlignmentModel : SymmetrizedWordAlignmentModel, ITransductiveWordAlignmentModel
4+
{
5+
private readonly ThotWordAlignmentModel _directWordAlignmentModel;
6+
private readonly ThotWordAlignmentModel _inverseWordAlignmentModel;
7+
8+
public ThotSymmetrizedWordAlignmentModel(
9+
ThotWordAlignmentModel directWordAlignmentModel,
10+
ThotWordAlignmentModel inverseWordAlignmentModel
11+
)
12+
: base(directWordAlignmentModel, inverseWordAlignmentModel)
13+
{
14+
_directWordAlignmentModel = directWordAlignmentModel;
15+
_inverseWordAlignmentModel = inverseWordAlignmentModel;
16+
}
17+
18+
public bool EmitTrainingAlignments
19+
{
20+
get => _directWordAlignmentModel.EmitTrainingAlignments;
21+
set
22+
{
23+
_directWordAlignmentModel.EmitTrainingAlignments = value;
24+
_inverseWordAlignmentModel.EmitTrainingAlignments = value;
25+
}
26+
}
27+
28+
public int TrainingAlignmentCount => _directWordAlignmentModel.TrainingAlignmentCount;
29+
30+
public WordAlignmentMatrix GetTrainingAlignment(int n)
31+
{
32+
WordAlignmentMatrix bestMatrix = _directWordAlignmentModel.GetTrainingAlignment(n);
33+
if (Heuristic == SymmetrizationHeuristic.None)
34+
return bestMatrix;
35+
36+
WordAlignmentMatrix invMatrix = _inverseWordAlignmentModel.GetTrainingAlignment(n);
37+
invMatrix.Transpose();
38+
39+
// Skip the combine when the matrices are degenerate or their dimensions don't
40+
// line up (e.g. an out-of-range n, or a pair filtered out of training in only
41+
// one direction): the heuristic operations require matching dimensions.
42+
if (
43+
bestMatrix.RowCount == 0
44+
|| bestMatrix.ColumnCount == 0
45+
|| invMatrix.RowCount != bestMatrix.RowCount
46+
|| invMatrix.ColumnCount != bestMatrix.ColumnCount
47+
)
48+
{
49+
return bestMatrix;
50+
}
51+
52+
bestMatrix.SymmetrizeWith(invMatrix, Heuristic);
53+
return bestMatrix;
54+
}
55+
}
56+
}

‎src/SIL.Machine.Translation.Thot/ThotWordAlignmentModel.cs‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,10 @@
1313

1414
namespace SIL.Machine.Translation.Thot
1515
{
16-
public abstract class ThotWordAlignmentModel : DisposableBase, IIbm1WordAlignmentModel
16+
public abstract class ThotWordAlignmentModel
17+
: DisposableBase,
18+
ITransductiveWordAlignmentModel,
19+
IIbm1WordAlignmentModel
1720
{
1821
public static ThotWordAlignmentModel Create(ThotWordAlignmentModelType type)
1922
{
@@ -156,6 +159,28 @@ public void Save()
156159
Thot.swAlignModel_save(Handle, _prefFileName);
157160
}
158161

162+
public bool EmitTrainingAlignments { get; set; }
163+
164+
public int TrainingAlignmentCount => (int)Thot.swAlignModel_getNumSentencePairs(Handle);
165+
166+
public WordAlignmentMatrix GetTrainingAlignment(int n)
167+
{
168+
CheckDisposed();
169+
IntPtr nativeMatrix = Thot.AllocNativeMatrix(_sourceWords.Count, _targetWords.Count);
170+
171+
uint iLen = (uint)_sourceWords.Count;
172+
uint jLen = (uint)_targetWords.Count;
173+
try
174+
{
175+
Thot.swAlignModel_getTrainingAlignment(Handle, (uint)n, nativeMatrix, ref iLen, ref jLen);
176+
return Thot.ConvertNativeMatrixToWordAlignmentMatrix(nativeMatrix, iLen, jLen);
177+
}
178+
finally
179+
{
180+
Thot.FreeNativeMatrix(nativeMatrix, iLen);
181+
}
182+
}
183+
159184
public double GetTranslationScore(string sourceWord, string targetWord)
160185
{
161186
return GetTranslationProbability(sourceWord, targetWord);
@@ -316,7 +341,7 @@ private class Trainer : ThotWordAlignmentModelTrainer
316341
private readonly ThotWordAlignmentModel _model;
317342

318343
public Trainer(ThotWordAlignmentModel model, IParallelTextCorpus corpus)
319-
: base(model.Type, corpus, model._prefFileName, model.Parameters)
344+
: base(model.Type, corpus, model._prefFileName, model.Parameters, model.EmitTrainingAlignments)
320345
{
321346
_model = model;
322347
CloseOnDispose = false;

‎src/SIL.Machine.Translation.Thot/ThotWordAlignmentModelTrainer.cs‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,10 @@ public ThotWordAlignmentModelTrainer(
2626
string sourceFileName,
2727
string targetFileName,
2828
string prefFileName,
29-
ThotWordAlignmentParameters parameters = null
29+
ThotWordAlignmentParameters parameters = null,
30+
bool emitTrainingAlignments = false
3031
)
31-
: this(modelType, null, prefFileName, parameters)
32+
: this(modelType, null, prefFileName, parameters, emitTrainingAlignments)
3233
{
3334
_sourceFileName = sourceFileName;
3435
_targetFileName = targetFileName;
@@ -38,7 +39,8 @@ public ThotWordAlignmentModelTrainer(
3839
ThotWordAlignmentModelType modelType,
3940
IParallelTextCorpus corpus,
4041
string prefFileName,
41-
ThotWordAlignmentParameters parameters = null
42+
ThotWordAlignmentParameters parameters = null,
43+
bool emitTrainingAlignments = false
4244
)
4345
{
4446
_prefFileName = prefFileName;
@@ -47,6 +49,8 @@ public ThotWordAlignmentModelTrainer(
4749
if (parameters == null)
4850
parameters = new ThotWordAlignmentParameters();
4951

52+
EmitTrainingAlignments = emitTrainingAlignments;
53+
5054
_models = new List<(IntPtr, int)>();
5155
if (modelType == ThotWordAlignmentModelType.FastAlign)
5256
{
@@ -197,6 +201,8 @@ public ThotWordAlignmentModelTrainer(
197201

198202
public TrainStats Stats { get; } = new TrainStats();
199203

204+
public bool EmitTrainingAlignments { get; }
205+
200206
public int MaxCorpusCount { get; set; } = int.MaxValue;
201207

202208
public Task TrainAsync(IProgress<ProgressStatus> progress = null, CancellationToken cancellationToken = default)
@@ -243,6 +249,14 @@ void Report() =>
243249
Report();
244250
cancellationToken.ThrowIfCancellationRequested();
245251

252+
if (EmitTrainingAlignments)
253+
{
254+
// Retain the alignments computed during training so that they can be returned without a
255+
// separate inference pass. Only the final (most refined) model's alignments are needed,
256+
// since that is the model used for inference.
257+
Thot.swAlignModel_setEmitTrainingAlignments(Handle, true);
258+
}
259+
246260
int trainedSegmentCount = 0;
247261
foreach ((IntPtr handle, int storedIterationCount) in _models)
248262
{

‎src/SIL.Machine/Translation/ITransductiveWordAlignmentModel.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
namespace SIL.Machine.Translation
2+
{
3+
public interface ITransductiveWordAlignmentModel
4+
{
5+
int TrainingAlignmentCount { get; }
6+
WordAlignmentMatrix GetTrainingAlignment(int n);
7+
}
8+
}

0 commit comments

Comments
 (0)