preprocessor.processing source code

1 /**
2  * Authors:
3  *  Mike Bierlee, m.bierlee@lostmoment.com
4  * Copyright: 2023 Mike Bierlee
5  * License:
6  *  This software is licensed under the terms of the MIT license.
7  *  The full terms of the license can be found in the LICENSE.txt file.
8  */
9 
10 module preprocessor.processing;
11 
12 import preprocessor.artifacts : BuildContext, PreprocessException, ParseException, FileMacro, LineMacro, MacroMap,
13     builtInMacros;
14 import preprocessor.parsing : ParseContext, parse, collect, DirectiveStart, MacroStartEnd, skipWhiteSpaceTillEol, peek,
15     replaceStartToEnd, clearStartToEnd, endOfLineDelims, peekLast, seekNextDirective, calculateLineColumn, seekNext,
16     collectTillString;
17 import preprocessor.debugging;
18 
19 import std.conv : to;
20 import std.path : dirName;
21 import std.algorithm : canFind;
22 import std.string : toLower, startsWith, endsWith, strip;
23 import std.array : replaceInPlace;
24 
25 private enum IncludeDirective = "include";
26 private enum IfDirective = "if";
27 private enum IfDefDirective = "ifdef";
28 private enum IfNDefDirective = "ifndef";
29 private enum ElIfDirective = "elif";
30 private enum ElseDirective = "else";
31 private enum EndIfDirective = "endif";
32 private enum DefineDirective = "define";
33 private enum UndefDirective = "undef";
34 private enum ErrorDirective = "error";
35 private enum PragmaDirective = "pragma";
36 
37 private enum PragmaOnceExtension = "once";
38 
39 private static const string[] conditionalTerminators = [
40     ElIfDirective, ElseDirective, EndIfDirective
41 ];
42 
43 package void processFile(
44     const string name,
45     const ref string inSource,
46     const ref BuildContext buildCtx,
47     ref MacroMap macros,
48     ref string[] guardedInclusions,
49     out string outSource,
50     const uint currentInclusionDepth = 0
51 ) {
52     macros[FileMacro] = name;
53     macros[LineMacro] = "true"; // For #if eval
54 
55     ParseContext parseCtx;
56     parseCtx.name = name;
57     parseCtx.source = inSource;
58     parseCtx.macros = macros;
59     parseCtx.guardedInclusions = guardedInclusions;
60     parseCtx.inclusionDepth = currentInclusionDepth;
61 
62     bool foundMacroTokenBefore = false;
63     parse(parseCtx, (const char chr, out bool stop) {
64         if (chr == DirectiveStart) {
65             foundMacroTokenBefore = false;
66             parseCtx.replaceStart = parseCtx.codePos - 1;
67             parseCtx.directive = parseCtx.collect();
68             processDirective(parseCtx, buildCtx);
69 
70             parseCtx.directive = "";
71             parseCtx.replaceStart = 0;
72             parseCtx.replaceEnd = 0;
73         } else if (chr == MacroStartEnd) {
74             if (foundMacroTokenBefore) {
75                 expandMacro(parseCtx);
76                 foundMacroTokenBefore = false;
77             } else {
78                 foundMacroTokenBefore = true;
79             }
80         } else {
81             foundMacroTokenBefore = false;
82         }
83     });
84 
85     macros = parseCtx.macros;
86     guardedInclusions = parseCtx.guardedInclusions;
87     outSource = parseCtx.source;
88 }
89 
90 private void processDirective(ref ParseContext parseCtx, const ref BuildContext buildCtx) {
91     switch (parseCtx.directive) {
92     case IncludeDirective:
93         if (buildCtx.enableIncludeDirectives) {
94             processInclude(parseCtx, buildCtx);
95         }
96         break;
97 
98     case IfDirective:
99     case IfDefDirective:
100     case IfNDefDirective:
101         if (buildCtx.enableConditionalDirectives) {
102             processConditionalDirective(parseCtx, parseCtx.directive);
103         }
104         break;
105 
106     case DefineDirective:
107         if (buildCtx.enableMacroDefineDirectives) {
108             processDefineDirective(parseCtx);
109         }
110         break;
111 
112     case UndefDirective:
113         if (buildCtx.enableMacroUndefineDirectives) {
114             processUndefDirective(parseCtx);
115         }
116         break;
117 
118     case EndIfDirective:
119         processUnexpectedConditional(parseCtx, buildCtx);
120         break;
121 
122     case ElseDirective:
123         processUnexpectedConditional(parseCtx, buildCtx);
124         break;
125 
126     case ElIfDirective:
127         processUnexpectedConditional(parseCtx, buildCtx);
128         break;
129 
130     case ErrorDirective:
131         if (buildCtx.enableErrorDirectives) {
132             processErrorDirective(parseCtx);
133         }
134         break;
135 
136     case PragmaDirective:
137         if (buildCtx.enablePragmaDirectives) {
138             processPragmaDirective(parseCtx);
139         }
140         break;
141 
142     default:
143         // Ignore directive. It may be of semantic importance to the source in another way.
144     }
145 }
146 
147 private void processInclude(ref ParseContext parseCtx, const ref BuildContext buildCtx) {
148     if (parseCtx.inclusionDepth >= buildCtx.inclusionLimit) {
149         throw new PreprocessException(parseCtx, "Inclusions has exceeded the limit of " ~
150                 buildCtx.inclusionLimit.to!string ~ ". Adjust BuildContext.inclusionLimit to increase.");
151     }
152 
153     parseCtx.codePos -= 1;
154     parseCtx.skipWhiteSpaceTillEol();
155     char startChr = parseCtx.peek;
156     bool absoluteInclusion;
157     if (startChr == '"') {
158         absoluteInclusion = false;
159     } else if (startChr == '<') {
160         absoluteInclusion = true;
161     } else {
162         throw new ParseException(parseCtx, "Failed to parse include directive: Expected \" or <.");
163     }
164 
165     parseCtx.codePos += 1;
166     const string includeName = parseCtx.collect(['"', '>']);
167     parseCtx.replaceEnd = parseCtx.codePos;
168 
169     auto includeSource = includeName in buildCtx.sources;
170     if (includeSource is null && !absoluteInclusion) {
171         string currentDir = parseCtx.name.dirName;
172         includeSource = currentDir ~ "/" ~ includeName in buildCtx.sources;
173     }
174 
175     if (includeSource is null) {
176         throw new PreprocessException(parseCtx, parseCtx.replaceStart, "Failed to include '" ~ includeName ~ "': It does not exist.");
177     }
178 
179     if (parseCtx.guardedInclusions.canFind(includeName)) {
180         parseCtx.clearStartToEnd();
181         return;
182     }
183 
184     string processedIncludeSource;
185     string[] guardedInclusions = parseCtx.guardedInclusions;
186     processFile(
187         includeName,
188         *includeSource,
189         buildCtx,
190         parseCtx.macros,
191         guardedInclusions,
192         processedIncludeSource,
193         parseCtx.inclusionDepth + 1
194     );
195 
196     parseCtx.macros[FileMacro] = parseCtx.name;
197     parseCtx.guardedInclusions = guardedInclusions;
198     parseCtx.replaceStartToEnd(processedIncludeSource);
199 }
200 
201 private void processConditionalDirective(ref ParseContext parseCtx, const string directiveName) {
202     bool negate = directiveName == IfNDefDirective;
203     bool onlyCheckExistence = directiveName != IfDirective;
204     processConditionalDirective(parseCtx, negate, onlyCheckExistence);
205 }
206 
207 private void processConditionalDirective(ref ParseContext parseCtx, const bool negate, const bool onlyCheckExistence) {
208     auto startOfConditionalBlock = parseCtx.replaceStart;
209     parseCtx.codePos -= 1;
210     parseCtx.skipWhiteSpaceTillEol();
211 
212     enum ConditionalBlockStartDirective = "startconditional";
213     auto conditionalDirective = ConditionalBlockStartDirective;
214     bool acceptedBody = false;
215     bool processedElse = false;
216     while (conditionalDirective != EndIfDirective) {
217         if (conditionalDirective == ConditionalBlockStartDirective || conditionalDirective == ElIfDirective) {
218             bool isTrue = evaluateCondition(parseCtx, negate, onlyCheckExistence);
219             if (isTrue && !acceptedBody) {
220                 parseCtx.acceptConditionalBody();
221                 acceptedBody = true;
222             } else {
223                 parseCtx.rejectConditionalBody();
224             }
225         } else if (conditionalDirective == ElseDirective) {
226             if (processedElse) {
227                 throw new ParseException(parseCtx, "#else directive defined multiple times. Only one #else block is allowed.");
228             }
229 
230             if (acceptedBody) {
231                 parseCtx.rejectConditionalBody();
232             } else {
233                 parseCtx.acceptConditionalBody();
234             }
235 
236             processedElse = true;
237         }
238 
239         parseCtx.replaceStart = parseCtx.codePos - 1;
240         conditionalDirective = parseCtx.collect();
241     }
242 
243     parseCtx.replaceEnd = parseCtx.codePos;
244     parseCtx.clearStartToEnd();
245 
246     parseCtx.codePos = startOfConditionalBlock;
247 }
248 
249 private void processDefineDirective(ref ParseContext parseCtx) {
250     auto macroName = parseCtx.collect();
251     if (macroName.length == 0) {
252         throw new ParseException(parseCtx, "#define directive is missing name of macro.");
253     }
254 
255     assertNotBuiltinMacro(parseCtx, macroName);
256 
257     string macroValue = null;
258     auto isEndOfDefinition = endOfLineDelims.canFind(parseCtx.peekLast);
259     if (!isEndOfDefinition) {
260         macroValue = parseCtx.collect(endOfLineDelims).strip;
261         if (macroValue[0] == '"' && macroValue[$ - 1] == '"') {
262             macroValue = macroValue[1 .. $ - 1];
263         }
264     }
265 
266     parseCtx.macros[macroName] = macroValue;
267     parseCtx.replaceEnd = parseCtx.codePos;
268     parseCtx.clearStartToEnd();
269 }
270 
271 private void processUndefDirective(ref ParseContext parseCtx) {
272     auto macroName = parseCtx.collect();
273     if (macroName.length == 0) {
274         throw new ParseException(parseCtx, "#undef directive is missing name of macro.");
275     }
276 
277     assertNotBuiltinMacro(parseCtx, macroName);
278 
279     parseCtx.macros.remove(macroName);
280     parseCtx.replaceEnd = parseCtx.codePos;
281     parseCtx.clearStartToEnd();
282 }
283 
284 private void processErrorDirective(ref ParseContext parseCtx) {
285     parseCtx.seekNext('"');
286     auto errorMessage = parseCtx.collect(endOfLineDelims ~ '"');
287     throw new PreprocessException(parseCtx, errorMessage);
288 }
289 
290 private void processPragmaDirective(ref ParseContext parseCtx) {
291     auto extensionName = parseCtx.collect();
292     if (extensionName != PragmaOnceExtension) {
293         throw new PreprocessException(parseCtx, "Pragma extension '" ~ extensionName ~ "' is unsupported.");
294     }
295 
296     parseCtx.guardedInclusions ~= parseCtx.name;
297     parseCtx.replaceEnd = parseCtx.codePos;
298     parseCtx.clearStartToEnd();
299 }
300 
301 private void processUnexpectedConditional(const ref ParseContext parseCtx, const ref BuildContext buildCtx) {
302     if (buildCtx.enableConditionalDirectives && !buildCtx.ignoreUnmatchedConditionalDirectives) {
303         throw new ParseException(parseCtx, "#" ~ parseCtx.directive ~ " directive found without accompanying starting conditional (#if/#ifdef)");
304     }
305 }
306 
307 private bool evaluateCondition(ref ParseContext parseCtx, const bool negate, const bool onlyCheckExistence) {
308     auto expression = parseCtx.collect();
309     if (expression.startsWith("__") && expression.endsWith("__")) {
310         expression = expression[2 .. $ - 2];
311     }
312 
313     auto macroValue = expression in parseCtx.macros;
314     bool isTrue = macroValue !is null;
315     if (!onlyCheckExistence) {
316         isTrue = isTrue && *macroValue != "0" && *macroValue != null
317             && (*macroValue).toLower != "false";
318     }
319 
320     if (negate) {
321         isTrue = !isTrue;
322     }
323 
324     return isTrue;
325 }
326 
327 private void acceptConditionalBody(ref ParseContext parseCtx) {
328     parseCtx.replaceEnd = parseCtx.codePos;
329     parseCtx.clearStartToEnd();
330     parseCtx.seekNextDirective(conditionalTerminators);
331 }
332 
333 private void rejectConditionalBody(ref ParseContext parseCtx) {
334     parseCtx.seekNextDirective(conditionalTerminators);
335     parseCtx.replaceEnd = parseCtx.codePos;
336     parseCtx.clearStartToEnd();
337 }
338 
339 private void expandMacro(ref ParseContext parseCtx) {
340     auto macroStart = parseCtx.codePos - 2;
341     auto macroName = parseCtx.collectTillString("__");
342     auto macroEnd = parseCtx.codePos;
343     if (parseCtx.peek == MacroStartEnd) {
344         macroEnd += 1;
345     }
346 
347     string macroValue;
348     if (macroName == LineMacro) {
349         ulong line, column;
350         calculateLineColumn(parseCtx, line, column);
351         macroValue = line.to!string;
352     } else {
353         auto macroValuePtr = macroName in parseCtx.macros;
354         if (macroValuePtr is null) {
355             throw new ParseException(parseCtx, "Cannot expand macro __" ~ macroName ~ "__, it is undefined.");
356         }
357 
358         macroValue = *macroValuePtr;
359     }
360 
361     parseCtx.source.replaceInPlace(macroStart, macroEnd, macroValue);
362     parseCtx.codePos = macroStart + macroValue.length;
363 }
364 
365 private void assertNotBuiltinMacro(ref ParseContext parseCtx, string macroName) {
366     if (builtInMacros.canFind(macroName)) {
367         throw new PreprocessException(parseCtx, "Cannot use macro name '" ~ macroName ~ "', it is a built-in macro.");
368     }
369 }