Browse Source

refactor a bit

feature/dependency-graph
Martin Dørum 3 years ago
parent
commit
4cc6725f7e
7 changed files with 278 additions and 229 deletions
  1. 2
    0
      .gitignore
  2. 2
    2
      Makefile
  3. 219
    0
      src/build.cc
  4. 24
    0
      src/build.h
  5. 17
    227
      src/main.cc
  6. 13
    0
      src/sys.cc
  7. 1
    0
      src/sys.h

+ 2
- 0
.gitignore View File

@@ -1 +1,3 @@
/build
/bbbuild
/compile_commands.json

+ 2
- 2
Makefile View File

@@ -1,8 +1,8 @@
SRCS = \
src/BBParser.cc src/SourceFile.cc src/toolchain.cc src/globals.cc \
src/logger.cc src/sys.cc src/parallel.cc src/main.cc
src/logger.cc src/sys.cc src/parallel.cc src/build.cc src/main.cc
HDRS = src/BBParser.h src/SourceFile.h src/toolchain.h src/globals.h \
src/logger.h src/sys.h src/parallel.h
src/logger.h src/sys.h src/parallel.h src/build.h
BUILD = build
OBJS = $(patsubst %,$(BUILD)/%.o,$(SRCS))
CFLAGS = -g -Wall -Wextra -Wno-unused-parameter

+ 219
- 0
src/build.cc View File

@@ -0,0 +1,219 @@
#include "build.h"

#include <fstream>
#include <sstream>
#include <mutex>
#include <unordered_set>

#include "sys.h"
#include "parallel.h"
#include "toolchain.h"
#include "logger.h"

namespace build {

static void findSourcesInDir(std::string dir, std::vector<SourceFile> &sources,
BBParser::Variables vars);

static void findSources(std::string dir, std::string name,
std::vector<SourceFile> &sources, BBParser::Variables vars) {
std::string path = dir + "/" + name;
sys::FileInfo finfo = sys::fileInfo(path);

if (finfo.isDir) {
// We don't want to send 'files'
BBParser::Variables subvars = vars;
subvars.erase("files");
findSourcesInDir(path, sources, std::move(subvars));
} else {
SourceFile::FileType type = SourceFile::fileTypeFrom(name);
if (type != SourceFile::FileType::UNKNOWN) {
sources.emplace_back(dir, name, type, vars);
}
}
}

static void findSourcesInDir(std::string dir, std::vector<SourceFile> &sources,
BBParser::Variables vars) {
std::ifstream buildFile(dir + "/build.bb");
std::vector<std::string> files;
bool hasFiles = false;

// Parse $dir/build.bb, see if it specifies a 'files' value.
if (buildFile.good()) {
BBParser parser(buildFile, BBParser::FLAG_NONE);
parser.parse(vars);

auto it = vars.find("files");
if (it != vars.end()) {
files = it->second;
hasFiles = true;
}
}

// If build.bb didn't specify 'files', we have to readdir
if (!hasFiles) {
sys::readDir(dir, files);
}

// Go through files
for (auto &ent: files) {
std::string path = dir + "/" + ent;
BBParser::Variables subvars = vars;
findSources(dir, ent, sources, vars);
}
}

static std::string findTargetName(const BuildConf &conf) {
auto it = conf.variables.find("target");
if (it != conf.variables.end() && it->second.size() != 0) {
return it->second[0];
}

for (auto &source: conf.sources) {
const std::vector<std::string> *ptr = source.variable("target");
if (ptr != nullptr && ptr->size() != 0) {
return (*ptr)[0];
}
}

return "target";
}

BuildConf readBuildConf(std::string outDir, int numJobs, const std::vector<std::string> &args) {
BuildConf conf;
conf.outDir = outDir;
conf.numJobs = numJobs;

// Read config from file
if (sys::fileExists("build.bb")) {
std::ifstream stream("build.bb");
BBParser parser(stream, BBParser::FLAG_NONE);
parser.parse(conf.variables);
}

// Read variables from args
for (auto &arg: args) {
std::istringstream stream(arg);
BBParser parser(stream, BBParser::FLAG_ONE_LINE);
parser.parse(conf.variables);
}

// Find source dirs
std::vector<std::string> sourceDirs;
{
auto it = conf.variables.find("files");
if (it == conf.variables.end()) {
sys::readDir(".", sourceDirs);
} else {
sourceDirs = it->second;
}
}

// Read configs from source dirs
for (std::string &dir: sourceDirs) {
findSources(".", dir, conf.sources, conf.variables);
}

conf.targetName = findTargetName(conf);

return conf;
}

bool compile(const BuildConf &conf) {
bool compiled = false;
std::mutex mut;
parallel::parallel(conf.numJobs, conf.sources, [&](const SourceFile &source) {
if (source.needsRecompile(conf.outDir)) {
source.compile(conf.outDir);
mut.lock();
compiled = true;
mut.unlock();
}
});

return compiled;
}

void link(const BuildConf &conf, toolchain::TargetType targetType) {
logger::log("Link " + conf.outDir + '/' + conf.targetName);

std::vector<std::string> ldFlags;
std::unordered_set<std::string> ldFlagsSet;
std::vector<std::string> ldLibs;
std::unordered_set<std::string> ldLibsSet;
std::vector<std::string> objects;

SourceFile::FileType type = SourceFile::FileType::C;

// Gather ldflags, ldlibs and objects from sources
std::vector<std::string> flags;
for (auto &source: conf.sources) {
if (source.type() != SourceFile::FileType::C) {
type = source.type();
}

flags = source.ldFlags();
for (auto &flag: flags) {
if (flag[0] == '-' && flag[1] == 'l') {
std::cerr << "Warning: -l flags should go in ldlibs "
<< "(at " << source.path() << ": " << flag << ")\n";
}

auto it = ldFlagsSet.find(flag);
if (it == ldFlagsSet.end()) {
ldFlagsSet.emplace(flag);
ldFlags.push_back(std::move(flag));
}
}

flags = source.ldLibs();
for (auto &flag: flags) {
auto it = ldLibsSet.find(flag);
if (it == ldLibsSet.end()) {
ldLibsSet.emplace(flag);
ldLibs.push_back(std::move(flag));
}
}

objects.push_back(std::move(source.objectPath(conf.outDir)));
}

flags.clear();

toolchain::link(conf.targetName, ldFlags, ldLibs, type, targetType, objects, conf.outDir);
}

void writeCompileCommands(const BuildConf &conf) {
sys::mkdirp(conf.outDir);
std::string cwd = sys::cwd();
std::ofstream os(conf.outDir + "/compile_commands.json");

os << "[\n\t";
bool first = true;
for (auto &source: conf.sources) {
std::string command;
for (auto arg: source.compileCommand(conf.outDir)) {
command += arg + ' ';
}
command.pop_back();

if (!first) {
os << ", ";
}

os << "{\n"
<< "\t\t\"directory\": \"" << cwd << "\"\n"
<< "\t\t\"command\": \"" << command << "\"\n"
<< "\t\t\"file\": \"" << source.path() << "\"\n"
<< "\t}";

first = false;
}

os << "\n]\n";

sys::symlink(conf.outDir + "/compile_commands.json", "compile_commands.json");
}

}

+ 24
- 0
src/build.h View File

@@ -0,0 +1,24 @@
#pragma once

#include <string>

#include "SourceFile.h"
#include "BBParser.h"
#include "toolchain.h"

namespace build {

struct BuildConf {
std::string outDir;
int numJobs;
BBParser::Variables variables;
std::string targetName;
std::vector<SourceFile> sources;
};

BuildConf readBuildConf(std::string outDir, int numJobs, const std::vector<std::string> &args);
bool compile(const BuildConf &conf);
void link(const BuildConf &conf, toolchain::TargetType targetType);
void writeCompileCommands(const BuildConf &conf);

}

+ 17
- 227
src/main.cc View File

@@ -1,12 +1,7 @@
//#bb ldlibs := -lpthread

#include <getopt.h>
#include <stdexcept>
#include <fstream>
#include <iostream>
#include <sstream>
#include <unordered_set>
#include <mutex>

#include "SourceFile.h"
#include "BBParser.h"
@@ -14,58 +9,7 @@
#include "toolchain.h"
#include "globals.h"
#include "logger.h"

static void findSourcesInDir(std::string dir, std::vector<SourceFile> &sources,
BBParser::Variables vars);

static void findSources(std::string dir, std::string name,
std::vector<SourceFile> &sources, BBParser::Variables vars) {
std::string path = dir + "/" + name;
sys::FileInfo finfo = sys::fileInfo(path);

if (finfo.isDir) {
// We don't want to send 'files'
BBParser::Variables subvars = vars;
subvars.erase("files");
findSourcesInDir(path, sources, std::move(subvars));
} else {
SourceFile::FileType type = SourceFile::fileTypeFrom(name);
if (type != SourceFile::FileType::UNKNOWN) {
sources.emplace_back(dir, name, type, vars);
}
}
}

static void findSourcesInDir(std::string dir, std::vector<SourceFile> &sources,
BBParser::Variables vars) {
std::ifstream buildFile(dir + "/build.bb");
std::vector<std::string> files;
bool hasFiles = false;

// Parse $dir/build.bb, see if it specifies a 'files' value.
if (buildFile.good()) {
BBParser parser(buildFile, BBParser::FLAG_NONE);
parser.parse(vars);

auto it = vars.find("files");
if (it != vars.end()) {
files = it->second;
hasFiles = true;
}
}

// If build.bb didn't specify 'files', we have to readdir
if (!hasFiles) {
sys::readDir(dir, files);
}

// Go through files
for (auto &ent: files) {
std::string path = dir + "/" + ent;
BBParser::Variables subvars = vars;
findSources(dir, ent, sources, vars);
}
}
#include "build.h"

static void printState(const std::vector<SourceFile> &sources) {
for (auto &source: sources) {
@@ -79,135 +23,18 @@ static void printState(const std::vector<SourceFile> &sources) {
}
}

static bool compile(const std::vector<SourceFile> &sources, const std::string &outDir, int jobs) {
bool compiled = false;
std::mutex mut;
parallel::parallel(jobs, sources, [&](const SourceFile &source) {
if (source.needsRecompile(outDir)) {
source.compile(outDir);
mut.lock();
compiled = true;
mut.unlock();
}
});

return compiled;
}

static void link(
const std::vector<SourceFile> &sources,
const std::string &outDir, const std::string &name,
toolchain::TargetType targetType) {
logger::log("Link " + outDir + '/' + name);

std::vector<std::string> ldFlags;
std::unordered_set<std::string> ldFlagsSet;
std::vector<std::string> ldLibs;
std::unordered_set<std::string> ldLibsSet;
std::vector<std::string> objects;

SourceFile::FileType type = SourceFile::FileType::C;

// Gather ldflags, ldlibs and objects from sources
std::vector<std::string> flags;
for (auto &source: sources) {
if (source.type() != SourceFile::FileType::C) {
type = source.type();
}

flags = source.ldFlags();
for (auto &flag: flags) {
if (flag[0] == '-' && flag[1] == 'l') {
std::cerr << "Warning: -l flags should go in ldlibs "
<< "(at " << source.path() << ": " << flag << ")\n";
}

auto it = ldFlagsSet.find(flag);
if (it == ldFlagsSet.end()) {
ldFlagsSet.emplace(flag);
ldFlags.push_back(std::move(flag));
}
}

flags = source.ldLibs();
for (auto &flag: flags) {
auto it = ldLibsSet.find(flag);
if (it == ldLibsSet.end()) {
ldLibsSet.emplace(flag);
ldLibs.push_back(std::move(flag));
}
}

objects.push_back(std::move(source.objectPath(outDir)));
}

flags.clear();

toolchain::link(name, ldFlags, ldLibs, type, targetType, objects, outDir);
}

static bool compileAndLink(
const std::vector<SourceFile> &sources, const std::string &outDir,
int jobs, const std::string &target, toolchain::TargetType targetType) {
std::string targetPath = toolchain::targetFilePath(targetType, target, outDir);
static bool compileAndLink(const build::BuildConf &conf, toolchain::TargetType targetType) {
std::string targetPath = toolchain::targetFilePath(
targetType, conf.targetName, conf.outDir);

if (compile(sources, outDir, jobs) || !sys::fileExists(targetPath)) {
link(sources, outDir, target, targetType);
if (build::compile(conf) || !sys::fileExists(targetPath)) {
build::link(conf, targetType);
return true;
}

return false;
}

static void printCompileCommands(
const std::vector<SourceFile> &sources, const std::string &outDir,
std::ostream &os) {
std::string cwd = sys::cwd();

os << "[\n\t";

bool first = true;
for (auto &source: sources) {
std::string command;
for (auto arg: source.compileCommand(outDir)) {
command += arg + ' ';
}
command.pop_back();

if (!first) {
os << ", ";
}

os << "{\n"
<< "\t\t\"directory\":\"" << cwd << "\"\n"
<< "\t\t\"command\":\"" << command << "\"\n"
<< "\t\t\"file\":\"" << source.path() << "\"\n"
<< "\t}";

first = false;
}

os << "\n]\n";
}

static std::string findTargetName(
const BBParser::Variables &vars,
const std::vector<SourceFile> &sources) {
auto it = vars.find("target");
if (it != vars.end() && it->second.size() != 0) {
return it->second[0];
}

for (auto &source: sources) {
const std::vector<std::string> *ptr = source.variable("target");
if (ptr != nullptr && ptr->size() != 0) {
return (*ptr)[0];
}
}

return "target";
}

int main(int argc, char **argv) {
logger::LogContext logCtx = logger::init();

@@ -217,7 +44,7 @@ int main(int argc, char **argv) {
std::string target = "";

enum class Action {
BUILD, PRINT_COMPILE_COMMANDS, PRINT_STATE,
BUILD, PRINT_STATE,
};
Action action = Action::BUILD;

@@ -229,7 +56,6 @@ int main(int argc, char **argv) {
{ "jobs", required_argument, NULL, 'j' },
{ "directory", required_argument, NULL, 'C' },
{ "target", required_argument, NULL, 't' },
{ "compile-commands", no_argument, NULL, 'c' },
{ "print-state", no_argument, NULL, 'p' },
{},
};
@@ -248,8 +74,6 @@ int main(int argc, char **argv) {
"Default: the number of cores in the machine.\n"
" -C, --directory <dir> "
"Change directory before doing anything else.\n"
" -c, --compile-commands "
"Write compile commands to stdout.\n"
" -p, --print-state "
"Print the state instead of building.\n";

@@ -287,10 +111,6 @@ int main(int argc, char **argv) {
workDir = optarg;
break;

case 'c':
action = Action::PRINT_COMPILE_COMMANDS;
break;

case 'p':
action = Action::PRINT_STATE;
break;
@@ -307,57 +127,27 @@ int main(int argc, char **argv) {
sys::chdir(workDir);
}

// Read config from file
BBParser::Variables variables;
if (sys::fileExists("build.bb")) {
std::ifstream stream("build.bb");
BBParser parser(stream, BBParser::FLAG_NONE);
parser.parse(variables);
}

// Read variables from argv
// Read everything related to the build configuration,
// finding all source files and reading their configs
std::vector<std::string> args;
while (optind < argc) {
std::istringstream stream(argv[optind++]);
BBParser parser(stream, BBParser::FLAG_ONE_LINE);
parser.parse(variables);
}

// Find source dirs
std::vector<std::string> sourceDirs;
{
auto it = variables.find("files");
if (it == variables.end()) {
sys::readDir(".", sourceDirs);
} else {
sourceDirs = it->second;
}
}

// Read configs from source dirs
std::vector<SourceFile> sources;
for (std::string &dir: sourceDirs) {
findSources(".", dir, sources, variables);
}

// Find target name
if (target.size() == 0) {
target = findTargetName(variables, sources);
args.push_back(argv[optind++]);
}
build::BuildConf conf = build::readBuildConf(std::move(outDir), jobs, args);
conf.numJobs = jobs;

switch (action) {
case Action::BUILD:
// TODO: Support more types than BINARY
if (!compileAndLink(sources, outDir, jobs, target, toolchain::TargetType::BINARY)) {
if (compileAndLink(conf, toolchain::TargetType::BINARY)) {
build::writeCompileCommands(conf);
} else {
logger::log("Nothing to do.");
}
break;

case Action::PRINT_COMPILE_COMMANDS:
printCompileCommands(sources, outDir, std::cout);
break;

case Action::PRINT_STATE:
printState(sources);
printState(conf.sources);
break;
}
}

+ 13
- 0
src/sys.cc View File

@@ -48,6 +48,7 @@ void mkdirp(const std::string &path) {
std::vector<std::string> argv;
argv.push_back("mkdir");
argv.push_back("-p");
argv.push_back("--");
argv.push_back(path);
execute(argv, nullptr, global::verbose >= 2);
}
@@ -184,4 +185,16 @@ std::string cwd() {
}
}

void symlink(const std::string &from, const std::string &to) {
// TODO: Implement this in C++ instead
std::vector<std::string> argv;
argv.push_back("ln");
argv.push_back("-s");
argv.push_back("-f");
argv.push_back("--");
argv.push_back(from);
argv.push_back(to);
sys::execute(argv, nullptr, global::verbose > 2);
}

}

+ 1
- 0
src/sys.h View File

@@ -18,5 +18,6 @@ void execute(const std::vector<std::string> &args, std::string *output, bool pri
void readDir(const std::string &path, std::vector<std::string> &files);
void chdir(const std::string &path);
std::string cwd();
void symlink(const std::string &from, const std::string &to);

}

Loading…
Cancel
Save