Skywalking-13：Skywalking模块加载机制

switchvov 06-27 117

前言：

当前各位老铁们对“让apache启动时可以动态加载文件吗为什么”都比较着重，咱们都想要学习一些“让apache启动时可以动态加载文件吗为什么”的相关知识。那么小编也在网摘上收集了一些对于“让apache启动时可以动态加载文件吗为什么””的相关内容，希望兄弟们能喜欢，兄弟们快快来了解一下吧！

模块加载机制基本概述

Module 是 Skywalking 在 OAP 提供的一种管理功能特性的机制。通过 Module 机制，可以方便的定义模块，并且可以提供多种实现，在配置文件中任意选择实现。

模块相关配置文件可以参考：Backend setup、Configuration Vocabulary

类图

Skywalking 中模块管理相关功能都在 org.apache.skywalking.oap.server.library.module 包下。

通过类图可以了解 Skywalking 模块机制大致分成如下几个模块：

模块配置： ApplicationConfiguration 、 ModuleConfiguration 、 ProviderConfigurationPS：刚好对应 application.yml 三层结构：模块->模块实现->某个模块实现的配置。模块定义类： ModuleDefine模块提供类： ModuleProvider服务： Service管理类： ModuleManager一些辅助类ModuleDefineHolder ：模块管理类需要实现的接口，提供查找模块相关功能ModuleProviderHolder ：模块定义类需要实现的接口，提供获取模块的服务类功能ModuleServiceHolder ：模块提供类需要实现的接口，提供注册服务实现、获取服务对象的功能ModuleConfig ：模块配置类，模块定义类会将ProviderConfiguration 映射为 ModuleConfigApplicationConfigLoader：ApplicationConfiguration 的辅助类，将 application.yml 配置文件加载到内存，设置 selector 对应的 Provider 的配置信息

类图源文件：Skywalking-Module.uml

源码解析ModuleDefine

package org.apache.skywalking.oap.server.library.module;import java.lang.reflect.Field;import java.util.Enumeration;import java.util.Properties;import java.util.ServiceLoader;import org.slf4j.Logger;import org.slf4j.LoggerFactory;// 模块定义public abstract class ModuleDefine implements ModuleProviderHolder {    private static final Logger LOGGER = LoggerFactory.getLogger(ModuleDefine.class);    // 模块实际的    private ModuleProvider loadedProvider = null;    private final String name;    public ModuleDefine(String name) {        this.name = name;    }    // 模块名    public final String name() {        return name;    }    // 实现类可以定义模块提供的服务类    public abstract Class[] services();    /**     * Run the prepare stage for the module, including finding all potential providers, and asking them to prepare.     *     * @param moduleManager of this module     * @param configuration of this module     * @throws ProviderNotFoundException when even don't find a single one providers.     */    // 准备阶段，找到configuration配置类对应的ModuleProvider对象，进行初始化操作    void prepare(        // 模块管理对象        ModuleManager moduleManager,         // 模块配置类        ApplicationConfiguration.ModuleConfiguration configuration,        // 模块提供类的服务加载器        ServiceLoader<ModuleProvider> moduleProviderLoader    ) throws ProviderNotFoundException, ServiceNotProvidedException, ModuleConfigException, ModuleStartException {        // 找到configuration配置类对应的ModuleProvider对象        for (ModuleProvider provider : moduleProviderLoader) {            if (!configuration.has(provider.name())) {                continue;            }            if (provider.module().equals(getClass())) {                if (loadedProvider == null) {                    loadedProvider = provider;                    loadedProvider.setManager(moduleManager);                    loadedProvider.setModuleDefine(this);                } else {                    throw new DuplicateProviderException(this.name() + " module has one " + loadedProvider.name() + "[" + loadedProvider.getClass().getName() + "] provider already, " + provider.name() + "[" + provider.getClass().getName() + "] is defined as 2nd provider.");                }            }        }        if (loadedProvider == null) {            throw new ProviderNotFoundException(this.name() + " module no provider found.");        }        // 复制提供类的配置文件至ModuleConfig对象        LOGGER.info("Prepare the {} provider in {} module.", loadedProvider.name(), this.name());        try {            copyProperties(                loadedProvider.createConfigBeanIfAbsent(),                 configuration.getProviderConfiguration(loadedProvider.name()),                 this.name(),                 loadedProvider.name()            );        } catch (IllegalAccessException e) {            throw new ModuleConfigException(this.name() + " module config transport to config bean failure.", e);        }        // 模块提供对象进入准备阶段        loadedProvider.prepare();    }    // 使用反射复制属性    private void copyProperties(ModuleConfig dest, Properties src, String moduleName, String providerName) throws IllegalAccessException {        if (dest == null) {            return;        }        Enumeration<?> propertyNames = src.propertyNames();        while (propertyNames.hasMoreElements()) {            String propertyName = (String) propertyNames.nextElement();            Class<? extends ModuleConfig> destClass = dest.getClass();            try {                Field field = getDeclaredField(destClass, propertyName);                field.setAccessible(true);                field.set(dest, src.get(propertyName));            } catch (NoSuchFieldException e) {                LOGGER.warn(propertyName + " setting is not supported in " + providerName + " provider of " + moduleName + " module");            }        }    }    private Field getDeclaredField(Class<?> destClass, String fieldName) throws NoSuchFieldException {        if (destClass != null) {            Field[] fields = destClass.getDeclaredFields();            for (Field field : fields) {                if (field.getName().equals(fieldName)) {                    return field;                }            }            return getDeclaredField(destClass.getSuperclass(), fieldName);        }        throw new NoSuchFieldException();    }    // 获取模块定义对应的Provider对象    @Override    public final ModuleProvider provider() throws DuplicateProviderException, ProviderNotFoundException {        if (loadedProvider == null) {            throw new ProviderNotFoundException("There is no module provider in " + this.name() + " module!");        }        return loadedProvider;    }}

ModuleProviderHolder

package org.apache.skywalking.oap.server.library.module;// 模块提供持有接口，通过该接口，可以获取模块Provider对象对应的Service持有接口，从而拿到模块Provider对象对应的服务对象public interface ModuleProviderHolder {    // 获取模块提供对象    ModuleServiceHolder provider() throws DuplicateProviderException, ProviderNotFoundException;}

ModuleProvider

package org.apache.skywalking.oap.server.library.module;import java.util.HashMap;import java.util.Map;import lombok.Setter;// 模块提供抽象类，所有的模块提供类都需要继承该抽象类// 一个模块定义可以配置多个模块提供类，通过在application.yml进行切换public abstract class ModuleProvider implements ModuleServiceHolder {    // 模块管理器    @Setter    private ModuleManager manager;    // 模块定义对象    @Setter    private ModuleDefine moduleDefine;    // 模块提供对应的服务对象map    private final Map<Class<? extends Service>, Service> services = new HashMap<>();    public ModuleProvider() {    }    protected final ModuleManager getManager() {        return manager;    }    // 获取服务提供实现类的name，需要子类实现    public abstract String name();    // 定义模块提供者所实现的模块定义类    public abstract Class<? extends ModuleDefine> module();    // 创建模块定义配置对象    public abstract ModuleConfig createConfigBeanIfAbsent();    // 准备阶段（初始化与其他模块无关的事情）    public abstract void prepare() throws ServiceNotProvidedException, ModuleStartException;    // 启动阶段（该阶段模块间可以互相操作）    public abstract void start() throws ServiceNotProvidedException, ModuleStartException;    // 完成后通知阶段（在所有模块成功启动后执行）    public abstract void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException;    // 该模块需要依赖的其他模块名    public abstract String[] requiredModules();    // 注册服务实现类    @Override    public final void registerServiceImplementation(Class<? extends Service> serviceType, Service service) throws ServiceNotProvidedException {        if (serviceType.isInstance(service)) {            this.services.put(serviceType, service);        } else {            throw new ServiceNotProvidedException(serviceType + " is not implemented by " + service);        }    }    // 确保所有服务被实现    void requiredCheck(Class<? extends Service>[] requiredServices) throws ServiceNotProvidedException {        if (requiredServices == null)            return;        for (Class<? extends Service> service : requiredServices) {            if (!services.containsKey(service)) {                throw new ServiceNotProvidedException("Service:" + service.getName() + " not provided");            }        }        if (requiredServices.length != services.size()) {            throw new ServiceNotProvidedException("The " + this.name() + " provider in " + moduleDefine.name() + " moduleDefine provide more service implementations than ModuleDefine requirements.");        }    }    // 获取服务实现对象    @Override    public @SuppressWarnings("unchecked")    <T extends Service> T getService(Class<T> serviceType) throws ServiceNotProvidedException {        Service serviceImpl = services.get(serviceType);        if (serviceImpl != null) {            return (T) serviceImpl;        }        throw new ServiceNotProvidedException("Service " + serviceType.getName() + " should not be provided, based on moduleDefine define.");    }    ModuleDefine getModule() {        return moduleDefine;    }    String getModuleName() {        return moduleDefine.name();    }}

ModuleConfig

package org.apache.skywalking.oap.server.library.module;// 模块配置类public abstract class ModuleConfig {}

ModuleServiceHolder

package org.apache.skywalking.oap.server.library.module;// 模块服务持有接口public interface ModuleServiceHolder {    // 注册服务实现对象    void registerServiceImplementation(Class<? extends Service> serviceType, Service service) throws ServiceNotProvidedException;    // 获取服务实现对象    <T extends Service> T getService(Class<T> serviceType) throws ServiceNotProvidedException;}

Service

package org.apache.skywalking.oap.server.library.module;// 服务接口public interface Service {}

ModuleDefineHolder

package org.apache.skywalking.oap.server.library.module;// 模块定义持有接口public interface ModuleDefineHolder {    // 判断是否有该模块    boolean has(String moduleName);    // 通过模块名获取模块定义对象    ModuleProviderHolder find(String moduleName) throws ModuleNotFoundRuntimeException;}

ModuleManager

package org.apache.skywalking.oap.server.library.module;import java.util.Arrays;import java.util.HashMap;import java.util.LinkedList;import java.util.Map;import java.util.ServiceLoader;// 模块管理类，管理模块的生命周期public class ModuleManager implements ModuleDefineHolder {    // 所有模块是否已经通过准备阶段    private boolean isInPrepareStage = true;    // 所有被加载的模块定义对象map    private final Map<String, ModuleDefine> loadedModules = new HashMap<>();    // 初始化所有配置的模块    public void init(ApplicationConfiguration applicationConfiguration) throws ModuleNotFoundException, ProviderNotFoundException, ServiceNotProvidedException, CycleDependencyException, ModuleConfigException, ModuleStartException {        // 获取配置类中的模块名        String[] moduleNames = applicationConfiguration.moduleList();        // SPI加载所有模块定义对象        ServiceLoader<ModuleDefine> moduleServiceLoader = ServiceLoader.load(ModuleDefine.class);        // SPI加载所有模块提供对象        ServiceLoader<ModuleProvider> moduleProviderLoader = ServiceLoader.load(ModuleProvider.class);        // 所有配置类中定义的模块，进行准备阶段        LinkedList<String> moduleList = new LinkedList<>(Arrays.asList(moduleNames));        for (ModuleDefine module : moduleServiceLoader) {            for (String moduleName : moduleNames) {                if (moduleName.equals(module.name())) {                    module.prepare(this, applicationConfiguration.getModuleConfiguration(moduleName), moduleProviderLoader);                    loadedModules.put(moduleName, module);                    moduleList.remove(moduleName);                }            }        }        // 准备阶段结束        isInPrepareStage = false;        if (moduleList.size() > 0) {            throw new ModuleNotFoundException(moduleList.toString() + " missing.");        }        // 根据模块提供对象中的requiredModules方法，确定模块的初始化顺序（被依赖的模块先行加载）        BootstrapFlow bootstrapFlow = new BootstrapFlow(loadedModules);        // 所有模块进入启动阶段        bootstrapFlow.start(this);        // 所有模块进入完成后通知阶段        bootstrapFlow.notifyAfterCompleted();    }    // 判断是否有该模块    @Override    public boolean has(String moduleName) {        return loadedModules.get(moduleName) != null;    }    // 通过模块名获取模块定义对象    @Override    public ModuleProviderHolder find(String moduleName) throws ModuleNotFoundRuntimeException {        assertPreparedStage();        ModuleDefine module = loadedModules.get(moduleName);        if (module != null)            return module;        throw new ModuleNotFoundRuntimeException(moduleName + " missing.");    }    // 断言是否还在准备阶段，如果还在准备阶段，则抛出异常    private void assertPreparedStage() {        if (isInPrepareStage) {            throw new AssertionError("Still in preparing stage.");        }    }}

BootstrapFlow

package org.apache.skywalking.oap.server.library.module;import java.util.ArrayList;import java.util.LinkedList;import java.util.List;import java.util.Map;import org.apache.skywalking.oap.server.library.util.CollectionUtils;import org.slf4j.Logger;import org.slf4j.LoggerFactory;// 根据模块提供对象中的requiredModules方法，确定模块的初始化顺序（被依赖的模块先行加载）class BootstrapFlow {    private static final Logger LOGGER = LoggerFactory.getLogger(BootstrapFlow.class);    private Map<String, ModuleDefine> loadedModules;    // 按依赖顺序排序的模块提供对象列表    private List<ModuleProvider> startupSequence;    BootstrapFlow(Map<String, ModuleDefine> loadedModules) throws CycleDependencyException, ModuleNotFoundException {        this.loadedModules = loadedModules;        startupSequence = new LinkedList<>();        // 被依赖的模块先行加载        makeSequence();    }    @SuppressWarnings("unchecked")    void start(        ModuleManager moduleManager) throws ModuleNotFoundException, ServiceNotProvidedException, ModuleStartException {        for (ModuleProvider provider : startupSequence) {            LOGGER.info("start the provider {} in {} module.", provider.name(), provider.getModuleName());            provider.requiredCheck(provider.getModule().services());            provider.start();        }    }    void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException {        for (ModuleProvider provider : startupSequence) {            provider.notifyAfterCompleted();        }    }    private void makeSequence() throws CycleDependencyException, ModuleNotFoundException {        List<ModuleProvider> allProviders = new ArrayList<>();        // 判断所有被依赖的模块是否存在        for (final ModuleDefine module : loadedModules.values()) {            String[] requiredModules = module.provider().requiredModules();            if (requiredModules != null) {                for (String requiredModule : requiredModules) {                    if (!loadedModules.containsKey(requiredModule)) {                        throw new ModuleNotFoundException(requiredModule + " module is required by " + module.provider().getModuleName() + "." + module.provider().name() + ", but not found.");                    }                }            }            allProviders.add(module.provider());        }        do {            int numOfToBeSequenced = allProviders.size();            for (int i = 0; i < allProviders.size(); i++) {                ModuleProvider provider = allProviders.get(i);                String[] requiredModules = provider.requiredModules();                if (CollectionUtils.isNotEmpty(requiredModules)) {                    // 是否所有依赖的模块都在startupSequence中                    boolean isAllRequiredModuleStarted = true;                    for (String module : requiredModules) {                        boolean exist = false;                        for (ModuleProvider moduleProvider : startupSequence) {                            if (moduleProvider.getModuleName().equals(module)) {                                exist = true;                                break;                            }                        }                        if (!exist) {                            isAllRequiredModuleStarted = false;                            break;                        }                    }                    // 所有依赖的模块都在startupSequence，则将该模块提供对象加入startupSequence                    if (isAllRequiredModuleStarted) {                        startupSequence.add(provider);                        allProviders.remove(i);                        i--;                    }                } else {                    // 如果该模块提供对象不依赖任何其他模块，则加入startupSequence                    startupSequence.add(provider);                    allProviders.remove(i);                    i--;                }            }            // 如果一次循环后，没有任何一个对象加入到startupSequence，则证明有循环依赖            if (numOfToBeSequenced == allProviders.size()) {                StringBuilder unSequencedProviders = new StringBuilder();                allProviders.forEach(provider -> unSequencedProviders.append(provider.getModuleName()).append("[provider=").append(provider.getClass().getName()).append("]\n"));                throw new CycleDependencyException("Exist cycle module dependencies in \n" + unSequencedProviders.substring(0, unSequencedProviders.length() - 1));            }        } while (allProviders.size() != 0); // 当提供对象列表不为空，则一直循环执行下去    }}

ApplicationConfiguration

package org.apache.skywalking.oap.server.library.module;import java.util.HashMap;import java.util.Properties;// OAP应用配置类public class ApplicationConfiguration {    // 模块定义配置map    private HashMap<String, ModuleConfiguration> modules = new HashMap<>();    // 模块配置名列表    public String[] moduleList() {        return modules.keySet().toArray(new String[0]);    }    // 添加模块定义配置    public ModuleConfiguration addModule(String moduleName) {        ModuleConfiguration newModule = new ModuleConfiguration();        modules.put(moduleName, newModule);        return newModule;    }    // 判断指定模块名是否存在模块定义配置map中    public boolean has(String moduleName) {        return modules.containsKey(moduleName);    }    // 获取模块定义配置    public ModuleConfiguration getModuleConfiguration(String name) {        return modules.get(name);    }    // 模块定义配置类    public static class ModuleConfiguration {        // 模块提供对象map        private HashMap<String, ProviderConfiguration> providers = new HashMap<>();        private ModuleConfiguration() {        }        // 获取模块提供配置        public Properties getProviderConfiguration(String name) {            return providers.get(name).getProperties();        }        // 是否存在模块提供配置        public boolean has(String name) {            return providers.containsKey(name);        }        // 添加模块提供配置        public ModuleConfiguration addProviderConfiguration(String name, Properties properties) {            ProviderConfiguration newProvider = new ProviderConfiguration(properties);            providers.put(name, newProvider);            return this;        }    }    // 模块提供配置类    public static class ProviderConfiguration {        // 模块提供属性        private Properties properties;        ProviderConfiguration(Properties properties) {            this.properties = properties;        }        private Properties getProperties() {            return properties;        }    }}

ApplicationConfigLoader

package org.apache.skywalking.oap.server.starter.config;import java.io.FileNotFoundException;import java.io.Reader;import java.util.HashSet;import java.util.Map;import java.util.Properties;import java.util.Set;import lombok.extern.slf4j.Slf4j;import org.apache.skywalking.apm.util.PropertyPlaceholderHelper;import org.apache.skywalking.oap.server.library.module.ApplicationConfiguration;import org.apache.skywalking.oap.server.library.module.ProviderNotFoundException;import org.apache.skywalking.oap.server.library.util.CollectionUtils;import org.apache.skywalking.oap.server.library.util.ResourceUtils;import org.yaml.snakeyaml.Yaml;// application.yml加载类, 三层结构:模块定义名.模块提供名.属性key@Slf4jpublic class ApplicationConfigLoader implements ConfigLoader<ApplicationConfiguration> {    // 当不配置模块提供者时，使用"-"    private static final String DISABLE_SELECTOR = "-";    // 该字段选择模块提供者    private static final String SELECTOR = "selector";    private final Yaml yaml = new Yaml();    @Override    public ApplicationConfiguration load() throws ConfigFileNotFoundException {        ApplicationConfiguration configuration = new ApplicationConfiguration();        this.loadConfig(configuration);        this.overrideConfigBySystemEnv(configuration);        return configuration;    }    @SuppressWarnings("unchecked")    private void loadConfig(ApplicationConfiguration configuration) throws ConfigFileNotFoundException {        try {            Reader applicationReader = ResourceUtils.read("application.yml");            Map<String, Map<String, Object>> moduleConfig = yaml.loadAs(applicationReader, Map.class);            if (CollectionUtils.isNotEmpty(moduleConfig)) {                selectConfig(moduleConfig);                moduleConfig.forEach((moduleName, providerConfig) -> {                    if (providerConfig.size() > 0) {                        log.info("Get a module define from application.yml, module name: {}", moduleName);                        ApplicationConfiguration.ModuleConfiguration moduleConfiguration = configuration.addModule(moduleName);                        providerConfig.forEach((providerName, config) -> {                            log.info("Get a provider define belong to {} module, provider name: {}", moduleName, providerName);                            final Map<String, ?> propertiesConfig = (Map<String, ?>) config;                            final Properties properties = new Properties();                            if (propertiesConfig != null) {                                propertiesConfig.forEach((propertyName, propertyValue) -> {                                    if (propertyValue instanceof Map) {                                        Properties subProperties = new Properties();                                        ((Map) propertyValue).forEach((key, value) -> {                                            subProperties.put(key, value);                                            replacePropertyAndLog(key, value, subProperties, providerName);                                        });                                        properties.put(propertyName, subProperties);                                    } else {                                        properties.put(propertyName, propertyValue);                                        replacePropertyAndLog(propertyName, propertyValue, properties, providerName);                                    }                                });                            }                            moduleConfiguration.addProviderConfiguration(providerName, properties);                        });                    } else {                        log.warn("Get a module define from application.yml, but no provider define, use default, module name: {}", moduleName);                    }                });            }        } catch (FileNotFoundException e) {            throw new ConfigFileNotFoundException(e.getMessage(), e);        }    }    private void replacePropertyAndLog(final Object propertyName, final Object propertyValue, final Properties target, final Object providerName) {        final String valueString = PropertyPlaceholderHelper.INSTANCE.replacePlaceholders(propertyValue + "", target);        if (valueString != null) {            if (valueString.trim().length() == 0) {                target.replace(propertyName, valueString);                log.info("Provider={} config={} has been set as an empty string", providerName, propertyName);            } else {                // Use YAML to do data type conversion.                final Object replaceValue = yaml.load(valueString);                if (replaceValue != null) {                    target.replace(propertyName, replaceValue);                    log.info("Provider={} config={} has been set as {}", providerName, propertyName, replaceValue.toString());                }            }        }    }    private void overrideConfigBySystemEnv(ApplicationConfiguration configuration) {        for (Map.Entry<Object, Object> prop : System.getProperties().entrySet()) {            overrideModuleSettings(configuration, prop.getKey().toString(), prop.getValue().toString());        }    }    private void selectConfig(final Map<String, Map<String, Object>> moduleConfiguration) {        final Set<String> modulesWithoutProvider = new HashSet<>();        for (final Map.Entry<String, Map<String, Object>> entry : moduleConfiguration.entrySet()) {            final String moduleName = entry.getKey();            final Map<String, Object> providerConfig = entry.getValue();            if (!providerConfig.containsKey(SELECTOR)) {                continue;            }            final String selector = (String) providerConfig.get(SELECTOR);            final String resolvedSelector = PropertyPlaceholderHelper.INSTANCE.replacePlaceholders(                selector, System.getProperties()            );            providerConfig.entrySet().removeIf(e -> !resolvedSelector.equals(e.getKey()));            if (!providerConfig.isEmpty()) {                continue;            }            if (!DISABLE_SELECTOR.equals(resolvedSelector)) {                throw new ProviderNotFoundException("no provider found for module " + moduleName + ", " + "if you're sure it's not required module and want to remove it, " + "set the selector to -");            }            // now the module can be safely removed            modulesWithoutProvider.add(moduleName);        }        moduleConfiguration.entrySet().removeIf(e -> {            final String module = e.getKey();            final boolean shouldBeRemoved = modulesWithoutProvider.contains(module);            if (shouldBeRemoved) {                log.info("Remove module {} without any provider", module);            }            return shouldBeRemoved;        });    }    private void overrideModuleSettings(ApplicationConfiguration configuration, String key, String value) {        int moduleAndConfigSeparator = key.indexOf('.');        if (moduleAndConfigSeparator <= 0) {            return;        }        String moduleName = key.substring(0, moduleAndConfigSeparator);        String providerSettingSubKey = key.substring(moduleAndConfigSeparator + 1);        ApplicationConfiguration.ModuleConfiguration moduleConfiguration = configuration.getModuleConfiguration(moduleName);        if (moduleConfiguration == null) {            return;        }        int providerAndConfigSeparator = providerSettingSubKey.indexOf('.');        if (providerAndConfigSeparator <= 0) {            return;        }        String providerName = providerSettingSubKey.substring(0, providerAndConfigSeparator);        String settingKey = providerSettingSubKey.substring(providerAndConfigSeparator + 1);        if (!moduleConfiguration.has(providerName)) {            return;        }        Properties providerSettings = moduleConfiguration.getProviderConfiguration(providerName);        if (!providerSettings.containsKey(settingKey)) {            return;        }        Object originValue = providerSettings.get(settingKey);        Class<?> type = originValue.getClass();        if (type.equals(int.class) || type.equals(Integer.class))            providerSettings.put(settingKey, Integer.valueOf(value));        else if (type.equals(String.class))            providerSettings.put(settingKey, value);        else if (type.equals(long.class) || type.equals(Long.class))            providerSettings.put(settingKey, Long.valueOf(value));        else if (type.equals(boolean.class) || type.equals(Boolean.class)) {            providerSettings.put(settingKey, Boolean.valueOf(value));        } else {            return;        }        log.info("The setting has been override by key: {}, value: {}, in {} provider of {} module through {}", settingKey, value, providerName, moduleName, "System.properties");    }}

ConfigLoader

package org.apache.skywalking.oap.server.starter.config;// 配置加载接口public interface ConfigLoader<T> {    T load() throws ConfigFileNotFoundException;}

OAPServerBootstrap

package org.apache.skywalking.oap.server.starter;import lombok.extern.slf4j.Slf4j;import org.apache.skywalking.oap.server.core.RunningMode;import org.apache.skywalking.oap.server.library.module.ApplicationConfiguration;import org.apache.skywalking.oap.server.library.module.ModuleManager;import org.apache.skywalking.oap.server.starter.config.ApplicationConfigLoader;import org.apache.skywalking.oap.server.starter.config.ConfigLoader;import org.apache.skywalking.oap.server.telemetry.TelemetryModule;import org.apache.skywalking.oap.server.telemetry.api.MetricsCreator;import org.apache.skywalking.oap.server.telemetry.api.MetricsTag;// OAP启动类，加载配置文件，初始化模块@Slf4jpublic class OAPServerBootstrap {    public static void start() {        String mode = System.getProperty("mode");        // 启动模式        RunningMode.setMode(mode);        ApplicationConfigLoader configLoader = new ApplicationConfigLoader();        ModuleManager manager = new ModuleManager();        try {            // 从配置文件中加载配置            ApplicationConfiguration applicationConfiguration = configLoader.load();            // 初始化模块            manager.init(applicationConfiguration);            // 将启动时间发送给Telemetry            manager.find(TelemetryModule.NAME)                   .provider()                   .getService(MetricsCreator.class)                   .createGauge("uptime", "oap server start up time", MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE)                   // Set uptime to second                   .setValue(System.currentTimeMillis() / 1000d);            if (RunningMode.isInitMode()) {                log.info("OAP starts up in init mode successfully, exit now...");                System.exit(0);            }        } catch (Throwable t) {            log.error(t.getMessage(), t);            System.exit(1);        }    }}

OAPServerStartUp

package org.apache.skywalking.oap.server.starter;// OAP启动类public class OAPServerStartUp {    public static void main(String[] args) {        OAPServerBootstrap.start();    }}

以 Skywalking OAP 启动流程分析模块加载机制时序图

源文件：OAPServerStartUp.sdt

案例：存储模块加载分析配置文件

从 application.yml 配置文件，可以看出。

模块是以三层结构来定义的：

第一层：模块定义名第二层：模块提供名/ selector第三层：模块提供配置信息/ selector 选择的模块提供配置

storage:  selector: ${SW_STORAGE:h2}  elasticsearch:    nameSpace: ${SW_NAMESPACE:""}    clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200}    # etc...  elasticsearch7:    nameSpace: ${SW_NAMESPACE:""}    clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200}    # etc...  h2:    driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource}    url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db;DB_CLOSE_DELAY=-1}    # etc...  mysql:    properties:      jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest"}      dataSource.user: ${SW_DATA_SOURCE_USER:root}      dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234}      # etc...  tidb:    properties:      jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:4000/tidbswtest"}      dataSource.user: ${SW_DATA_SOURCE_USER:root}      dataSource.password: ${SW_DATA_SOURCE_PASSWORD:""}      # etc...  influxdb:    url: ${SW_STORAGE_INFLUXDB_URL:}    user: ${SW_STORAGE_INFLUXDB_USER:root}    password: ${SW_STORAGE_INFLUXDB_PASSWORD:}    # etc...

加载配置

经过 org.apache.skywalking.oap.server.starter.config.ApplicationConfigLoader#load 的调用， org.apache.skywalking.oap.server.starter.OAPServerBootstrap#start 获取到了所有需要加载的模块，其中包括存储模块。

在 org.apache.skywalking.oap.server.starter.config.ApplicationConfigLoader#selectConfig 也通过 storage.selector=h2 ，存储模块只保留了 h2 的配置信息：

storage:    h2:    driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource}    url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db;DB_CLOSE_DELAY=-1}    # etc...

准备阶段

在org.apache.skywalking.oap.server.library.module.ModuleManager#init 中，通过 SPI 加载了模块定义对象，存储模块对应的定义类如下：

PS：可以看到定义了大量 Service 接口

package org.apache.skywalking.oap.server.core.storage;import org.apache.skywalking.oap.server.core.storage.cache.INetworkAddressAliasDAO;import org.apache.skywalking.oap.server.core.storage.management.UITemplateManagementDAO;import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskLogQueryDAO;import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDAO;import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO;import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO;import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO;import org.apache.skywalking.oap.server.core.storage.query.IBrowserLogQueryDAO;import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO;import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO;import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO;import org.apache.skywalking.oap.server.core.storage.query.ITopNRecordsQueryDAO;import org.apache.skywalking.oap.server.core.storage.query.ITopologyQueryDAO;import org.apache.skywalking.oap.server.core.storage.query.ITraceQueryDAO;import org.apache.skywalking.oap.server.library.module.ModuleDefine;/** * StorageModule provides the capabilities(services) to interact with the database. With different databases, this * module could have different providers, such as currently, H2, MySQL, ES, TiDB. */public class StorageModule extends ModuleDefine {    public static final String NAME = "storage";    public StorageModule() {        super(NAME);    }    @Override    public Class[] services() {        return new Class[]{                IBatchDAO.class,                StorageDAO.class,                IHistoryDeleteDAO.class,                INetworkAddressAliasDAO.class,                ITopologyQueryDAO.class,                IMetricsQueryDAO.class,                ITraceQueryDAO.class,                IMetadataQueryDAO.class,                IAggregationQueryDAO.class,                IAlarmQueryDAO.class,                ITopNRecordsQueryDAO.class,                ILogQueryDAO.class,                IProfileTaskQueryDAO.class,                IProfileTaskLogQueryDAO.class,                IProfileThreadSnapshotQueryDAO.class,                UITemplateManagementDAO.class,                IBrowserLogQueryDAO.class        };    }}

同时也会调用 org.apache.skywalking.oap.server.library.module.ModuleDefine#prepare 进入准备阶段

        String[] moduleNames = applicationConfiguration.moduleList();        ServiceLoader<ModuleDefine> moduleServiceLoader = ServiceLoader.load(ModuleDefine.class);        ServiceLoader<ModuleProvider> moduleProviderLoader = ServiceLoader.load(ModuleProvider.class);        LinkedList<String> moduleList = new LinkedList<>(Arrays.asList(moduleNames));        for (ModuleDefine module : moduleServiceLoader) {            for (String moduleName : moduleNames) {                if (moduleName.equals(module.name())) {                    module.prepare(this, applicationConfiguration.getModuleConfiguration(moduleName), moduleProviderLoader);                    loadedModules.put(moduleName, module);                    moduleList.remove(moduleName);                }            }        }

在 org.apache.skywalking.oap.server.library.module.ModuleDefine#prepare 会通过传入的配置，只匹配上配置文件选择的模块提供对象。

        for (ModuleProvider provider : moduleProviderLoader) {            if (!configuration.has(provider.name())) {                continue;            }            if (provider.module().equals(getClass())) {                if (loadedProvider == null) {                    loadedProvider = provider;                    loadedProvider.setManager(moduleManager);                    loadedProvider.setModuleDefine(this);                } else {                    throw new DuplicateProviderException(this.name() + " module has one " + loadedProvider.name() + "[" + loadedProvider.getClass().getName()                                                          + "] provider already, " + provider.name() + "[" + provider.getClass().getName() + "] is defined as 2nd provider.");                }            }        }        if (loadedProvider == null) {            throw new ProviderNotFoundException(this.name() + " module no provider found.");        }        LOGGER.info("Prepare the {} provider in {} module.", loadedProvider.name(), this.name());        try {            copyProperties(loadedProvider.createConfigBeanIfAbsent(), configuration.getProviderConfiguration(loadedProvider.name()), this.name(), loadedProvider.name());        } catch (IllegalAccessException e) {            throw new ModuleConfigException(this.name() + " module config transport to config bean failure.", e);        }        loadedProvider.prepare();

例如“配置文件”一节选择的h2 ，则加载的提供类为 org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.H2StorageProvider

它的 prepare 方法如下，可以看到：注册了所有 StorageModule 声明的 Service 接口

    @Override    public void prepare() throws ServiceNotProvidedException, ModuleStartException {        Properties settings = new Properties();        settings.setProperty("dataSourceClassName", config.getDriver());        settings.setProperty("dataSource.url", config.getUrl());        settings.setProperty("dataSource.user", config.getUser());        settings.setProperty("dataSource.password", config.getPassword());        h2Client = new JDBCHikariCPClient(settings);        this.registerServiceImplementation(IBatchDAO.class, new H2BatchDAO(h2Client));        this.registerServiceImplementation(            StorageDAO.class,            new H2StorageDAO(                getManager(), h2Client, config.getMaxSizeOfArrayColumn(), config.getNumOfSearchableValuesPerTag())        );        this.registerServiceImplementation(            INetworkAddressAliasDAO.class, new H2NetworkAddressAliasDAO(h2Client));        this.registerServiceImplementation(ITopologyQueryDAO.class, new H2TopologyQueryDAO(h2Client));        this.registerServiceImplementation(IMetricsQueryDAO.class, new H2MetricsQueryDAO(h2Client));        this.registerServiceImplementation(            ITraceQueryDAO.class, new H2TraceQueryDAO(                getManager(),                h2Client,                config.getMaxSizeOfArrayColumn(),                config.getNumOfSearchableValuesPerTag()            ));        this.registerServiceImplementation(IBrowserLogQueryDAO.class, new H2BrowserLogQueryDAO(h2Client));        this.registerServiceImplementation(            IMetadataQueryDAO.class, new H2MetadataQueryDAO(h2Client, config.getMetadataQueryMaxSize()));        this.registerServiceImplementation(IAggregationQueryDAO.class, new H2AggregationQueryDAO(h2Client));        this.registerServiceImplementation(IAlarmQueryDAO.class, new H2AlarmQueryDAO(h2Client));        this.registerServiceImplementation(            IHistoryDeleteDAO.class, new H2HistoryDeleteDAO(h2Client));        this.registerServiceImplementation(ITopNRecordsQueryDAO.class, new H2TopNRecordsQueryDAO(h2Client));        this.registerServiceImplementation(            ILogQueryDAO.class,            new H2LogQueryDAO(                h2Client,                getManager(),                config.getMaxSizeOfArrayColumn(),                config.getNumOfSearchableValuesPerTag()            )        );        this.registerServiceImplementation(IProfileTaskQueryDAO.class, new H2ProfileTaskQueryDAO(h2Client));        this.registerServiceImplementation(IProfileTaskLogQueryDAO.class, new H2ProfileTaskLogQueryDAO(h2Client));        this.registerServiceImplementation(            IProfileThreadSnapshotQueryDAO.class, new H2ProfileThreadSnapshotQueryDAO(h2Client));        this.registerServiceImplementation(UITemplateManagementDAO.class, new H2UITemplateManagementDAO(h2Client));    }

启动阶段

在org.apache.skywalking.oap.server.library.module.ModuleManager#init 通过调用 org.apache.skywalking.oap.server.library.module.BootstrapFlow#start 进入启动阶段

    void start(        ModuleManager moduleManager) throws ModuleNotFoundException, ServiceNotProvidedException, ModuleStartException {        for (ModuleProvider provider : startupSequence) {            LOGGER.info("start the provider {} in {} module.", provider.name(), provider.getModuleName());            provider.requiredCheck(provider.getModule().services());            provider.start();        }    }

例如“配置文件”一节选择的h2 ，则加载的提供类为 org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.H2StorageProvider

它的 start 方法如下，可以看到：启动 h2client 并监听 ModelCreator

    @Override    public void start() throws ServiceNotProvidedException, ModuleStartException {        final ConfigService configService = getManager().find(CoreModule.NAME)                                                        .provider()                                                        .getService(ConfigService.class);        final int numOfSearchableTracesTags = configService.getSearchableTracesTags().split(Const.COMMA).length;        if (numOfSearchableTracesTags * config.getNumOfSearchableValuesPerTag() > config.getMaxSizeOfArrayColumn()) {            throw new ModuleStartException("Size of searchableTracesTags[" + numOfSearchableTracesTags                                               + "] * numOfSearchableValuesPerTag[" + config.getNumOfSearchableValuesPerTag()                                               + "] > maxSizeOfArrayColumn[" + config.getMaxSizeOfArrayColumn()                                               + "]. Potential out of bound in the runtime.");        }        final int numOfSearchableLogsTags = configService.getSearchableLogsTags().split(Const.COMMA).length;        if (numOfSearchableLogsTags * config.getNumOfSearchableValuesPerTag() > config.getMaxSizeOfArrayColumn()) {            throw new ModuleStartException("Size of searchableLogsTags[" + numOfSearchableLogsTags                                               + "] * numOfSearchableValuesPerTag[" + config.getNumOfSearchableValuesPerTag()                                               + "] > maxSizeOfArrayColumn[" + config.getMaxSizeOfArrayColumn()                                               + "]. Potential out of bound in the runtime.");        }        MetricsCreator metricCreator = getManager().find(TelemetryModule.NAME)                                                   .provider()                                                   .getService(MetricsCreator.class);        HealthCheckMetrics healthChecker = metricCreator.createHealthCheckerGauge(            "storage_h2", MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE);        h2Client.registerChecker(healthChecker);        try {            h2Client.connect();            H2TableInstaller installer = new H2TableInstaller(                h2Client, getManager(), config.getMaxSizeOfArrayColumn(), config.getNumOfSearchableValuesPerTag());            getManager().find(CoreModule.NAME).provider().getService(ModelCreator.class).addModelListener(installer);        } catch (StorageException e) {            throw new ModuleStartException(e.getMessage(), e);        }    }

完成后通知阶段

在org.apache.skywalking.oap.server.library.module.ModuleManager#init 通过调用 org.apache.skywalking.oap.server.library.module.BootstrapFlow#notifyAfterCompleted 进入完成后通知阶段

    void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException {        for (ModuleProvider provider : startupSequence) {            provider.notifyAfterCompleted();        }    }

例如“配置文件”一节选择的h2 ，则加载的提供类为 org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.H2StorageProvider

它的 notifyAfterCompleted 方法如下，可以看到：不需要做什么

    @Override    public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException {    }

H2StorageProvider 完整源码

package org.apache.skywalking.oap.server.storage.plugin.jdbc.h2;// etc.../** * H2 Storage provider is for demonstration and preview only. I will find that haven't implemented several interfaces, * because not necessary, and don't consider about performance very much. * <p> * If someone wants to implement SQL-style database as storage, please just refer the logic. */@Slf4jpublic class H2StorageProvider extends ModuleProvider {    private H2StorageConfig config;    private JDBCHikariCPClient h2Client;    public H2StorageProvider() {        config = new H2StorageConfig();    }    @Override    public String name() {        return "h2";    }    @Override    public Class<? extends ModuleDefine> module() {        return StorageModule.class;    }    @Override    public ModuleConfig createConfigBeanIfAbsent() {        return config;    }    @Override    public void prepare() throws ServiceNotProvidedException, ModuleStartException {        Properties settings = new Properties();        settings.setProperty("dataSourceClassName", config.getDriver());        settings.setProperty("dataSource.url", config.getUrl());        settings.setProperty("dataSource.user", config.getUser());        settings.setProperty("dataSource.password", config.getPassword());        h2Client = new JDBCHikariCPClient(settings);        this.registerServiceImplementation(IBatchDAO.class, new H2BatchDAO(h2Client));        this.registerServiceImplementation(            StorageDAO.class,            new H2StorageDAO(                getManager(), h2Client, config.getMaxSizeOfArrayColumn(), config.getNumOfSearchableValuesPerTag())        );        this.registerServiceImplementation(            INetworkAddressAliasDAO.class, new H2NetworkAddressAliasDAO(h2Client));        this.registerServiceImplementation(ITopologyQueryDAO.class, new H2TopologyQueryDAO(h2Client));        this.registerServiceImplementation(IMetricsQueryDAO.class, new H2MetricsQueryDAO(h2Client));        this.registerServiceImplementation(            ITraceQueryDAO.class, new H2TraceQueryDAO(                getManager(),                h2Client,                config.getMaxSizeOfArrayColumn(),                config.getNumOfSearchableValuesPerTag()            ));        this.registerServiceImplementation(IBrowserLogQueryDAO.class, new H2BrowserLogQueryDAO(h2Client));        this.registerServiceImplementation(            IMetadataQueryDAO.class, new H2MetadataQueryDAO(h2Client, config.getMetadataQueryMaxSize()));        this.registerServiceImplementation(IAggregationQueryDAO.class, new H2AggregationQueryDAO(h2Client));        this.registerServiceImplementation(IAlarmQueryDAO.class, new H2AlarmQueryDAO(h2Client));        this.registerServiceImplementation(            IHistoryDeleteDAO.class, new H2HistoryDeleteDAO(h2Client));        this.registerServiceImplementation(ITopNRecordsQueryDAO.class, new H2TopNRecordsQueryDAO(h2Client));        this.registerServiceImplementation(            ILogQueryDAO.class,            new H2LogQueryDAO(                h2Client,                getManager(),                config.getMaxSizeOfArrayColumn(),                config.getNumOfSearchableValuesPerTag()            )        );        this.registerServiceImplementation(IProfileTaskQueryDAO.class, new H2ProfileTaskQueryDAO(h2Client));        this.registerServiceImplementation(IProfileTaskLogQueryDAO.class, new H2ProfileTaskLogQueryDAO(h2Client));        this.registerServiceImplementation(            IProfileThreadSnapshotQueryDAO.class, new H2ProfileThreadSnapshotQueryDAO(h2Client));        this.registerServiceImplementation(UITemplateManagementDAO.class, new H2UITemplateManagementDAO(h2Client));    }    @Override    public void start() throws ServiceNotProvidedException, ModuleStartException {        final ConfigService configService = getManager().find(CoreModule.NAME)                                                        .provider()                                                        .getService(ConfigService.class);        final int numOfSearchableTracesTags = configService.getSearchableTracesTags().split(Const.COMMA).length;        if (numOfSearchableTracesTags * config.getNumOfSearchableValuesPerTag() > config.getMaxSizeOfArrayColumn()) {            throw new ModuleStartException("Size of searchableTracesTags[" + numOfSearchableTracesTags                                               + "] * numOfSearchableValuesPerTag[" + config.getNumOfSearchableValuesPerTag()                                               + "] > maxSizeOfArrayColumn[" + config.getMaxSizeOfArrayColumn()                                               + "]. Potential out of bound in the runtime.");        }        final int numOfSearchableLogsTags = configService.getSearchableLogsTags().split(Const.COMMA).length;        if (numOfSearchableLogsTags * config.getNumOfSearchableValuesPerTag() > config.getMaxSizeOfArrayColumn()) {            throw new ModuleStartException("Size of searchableLogsTags[" + numOfSearchableLogsTags                                               + "] * numOfSearchableValuesPerTag[" + config.getNumOfSearchableValuesPerTag()                                               + "] > maxSizeOfArrayColumn[" + config.getMaxSizeOfArrayColumn()                                               + "]. Potential out of bound in the runtime.");        }        MetricsCreator metricCreator = getManager().find(TelemetryModule.NAME)                                                   .provider()                                                   .getService(MetricsCreator.class);        HealthCheckMetrics healthChecker = metricCreator.createHealthCheckerGauge(            "storage_h2", MetricsTag.EMPTY_KEY, MetricsTag.EMPTY_VALUE);        h2Client.registerChecker(healthChecker);        try {            h2Client.connect();            H2TableInstaller installer = new H2TableInstaller(                h2Client, getManager(), config.getMaxSizeOfArrayColumn(), config.getNumOfSearchableValuesPerTag());            getManager().find(CoreModule.NAME).provider().getService(ModelCreator.class).addModelListener(installer);        } catch (StorageException e) {            throw new ModuleStartException(e.getMessage(), e);        }    }    @Override    public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException {    }    @Override    public String[] requiredModules() {        return new String[] {CoreModule.NAME};    }}

总结

Skywalking 提供的模块机制是非常优良的设计，在工作中，如果有多个 N 选 1 的场景，是可以借鉴它的设计的。

参考文档Backend setupConfiguration Vocabulary

分享并记录所学所见

本文地址：http://www.longkongtuishu.com/ca287BAFsBlIBDV0.html

标签： #让apache启动时可以动态加载文件吗为什么